{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from brnn_model_fake_data import *\n",
    "import reader\n",
    "\n",
    "import subprocess\n",
    "import tensorflow as tf\n",
    "import import_folders\n",
    "import pickle_lib as pkl\n",
    "from graph_lib import gl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Global variables\n",
    "\"\"\"\n",
    "model_type = \"test\"\n",
    "data_path = \"../data/\"\n",
    "save_path = \"./saved_model/\"\n",
    "global_prior_pi = 0.25\n",
    "global_log_sigma1 = -1.0\n",
    "global_log_sigma2 = -7.0\n",
    "global_random_seed = 12\n",
    "global_num_gpus = 0\n",
    "\n",
    "\n",
    "# Model can be \"test\", \"small\", \"medium\", \"large\"\n",
    "model_select = \"test\"\n",
    "model_type = model_select\n",
    "#Put the path to the data here\n",
    "dat_path = \"../data\"\n",
    "\n",
    "#Put the path to where you want to save the training data\n",
    "sav_path = \"tensorboard/\"\n",
    "\n",
    "# The mixing degree for the prior gaussian mixture\n",
    "# As in Fortunato they report scanning\n",
    "# mix_pi \\in { 1/4, 1/2, 3/4 }\n",
    "mixing_pi = 0.25\n",
    "\n",
    "# As in Fortunato they report scanning\n",
    "# log sigma1 \\in { 0, -1, -2 }\n",
    "# log sigma2 \\in { -6, -7, -8 }\n",
    "prior_log_sigma1 = -1.0\n",
    "prior_log_sigma2 = -7.0\n",
    "\n",
    "\n",
    "class SmallConfig(object):\n",
    "    \"\"\"Small config.\"\"\"\n",
    "    init_scale = 0.1\n",
    "    learning_rate = 1.0\n",
    "    max_grad_norm = 5\n",
    "    num_layers = 2\n",
    "    num_steps = 20\n",
    "    hidden_size = 200\n",
    "    max_epoch = 4\n",
    "    max_max_epoch = 13\n",
    "    keep_prob = 1.0\n",
    "    lr_decay = 0.5\n",
    "    \n",
    "    batch_size = 20\n",
    "    vocab_size = 10000\n",
    "    \n",
    "    X_dim = 200 # Size of the embedding\n",
    "\n",
    "class MediumConfig(object):\n",
    "    \"\"\"\n",
    "    Medium config.\n",
    "    Slightly modified according to email.\n",
    "    \"\"\"\n",
    "    init_scale = 0.05\n",
    "    learning_rate = 1.0\n",
    "    max_grad_norm = 5\n",
    "    num_layers = 2\n",
    "    num_steps = 35\n",
    "    hidden_size = 650\n",
    "    max_epoch = 20\n",
    "    max_max_epoch = 70\n",
    "    keep_prob = 1.0\n",
    "    lr_decay = 0.9\n",
    "    batch_size = 20\n",
    "    vocab_size = 10000\n",
    "\n",
    "    X_dim = 50 # Size of the embedding\n",
    "    \n",
    "class LargeConfig(object):\n",
    "    \"\"\"Large config.\"\"\"\n",
    "    init_scale = 0.04\n",
    "    learning_rate = 1.0\n",
    "    max_grad_norm = 10\n",
    "    num_layers = 2\n",
    "    num_steps = 35\n",
    "    hidden_size = 1500\n",
    "    max_epoch = 14\n",
    "    max_max_epoch = 55\n",
    "    keep_prob = 0.35\n",
    "    lr_decay = 1 / 1.15\n",
    "    batch_size = 20\n",
    "    vocab_size = 10000\n",
    "\n",
    "    X_dim = 100 # Size of the embedding\n",
    "    \n",
    "class TestConfig(object):\n",
    "    \"\"\"Tiny config, for testing.\"\"\"\n",
    "    init_scale = 0.1\n",
    "    learning_rate = 0.5\n",
    "    max_grad_norm = 1\n",
    "    num_layers = 2\n",
    "    num_steps = 20\n",
    "    hidden_size = 16\n",
    "    max_epoch = 1\n",
    "    max_max_epoch = 20\n",
    "    keep_prob = 1.0\n",
    "    lr_decay = 0.9\n",
    "    batch_size = 1\n",
    "    \n",
    "    vocab_size = 10000\n",
    "\n",
    "    X_dim = 19 # Size of the embedding\n",
    "\n",
    "\n",
    "#    global_random_seed = set_random_seed\n",
    "    \n",
    "def get_config():\n",
    "    \"\"\"Get model config.\"\"\"\n",
    "    if model_type == \"small\":\n",
    "        config = SmallConfig()\n",
    "    elif model_type == \"medium\":\n",
    "        config = MediumConfig()\n",
    "    elif model_type == \"large\":\n",
    "        config = LargeConfig()\n",
    "    elif model_type == \"test\":\n",
    "        config = TestConfig()\n",
    "    else:\n",
    "        raise ValueError(\"Invalid model: %s\", model_type)\n",
    "\n",
    "    print (\"Model Type\")\n",
    "    print (model_type)\n",
    "    config.prior_pi = global_prior_pi\n",
    "    config.log_sigma1 = global_log_sigma1\n",
    "    config.log_sigma2 = global_log_sigma2\n",
    "\n",
    "    return config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test\n",
      "Model Type\n",
      "test\n",
      "Model Type\n",
      "test\n",
      "Number of total initial chains 20000\n",
      "Dimensionality of chains (num_step,X_dim) (20, 1)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print (model_type)\n",
    "\n",
    "####### Global data reading #########\n",
    "Ndivisions = 10;\n",
    "folder_data = \"./data/artificial/\"\n",
    "\n",
    "X_list = pkl.load_pickle(folder_data +\"X_values.pkl\",Ndivisions)\n",
    "Y_list = pkl.load_pickle(folder_data +\"Y_values.pkl\",Ndivisions)\n",
    "\n",
    "num_steps, X_dim = X_list[0].shape\n",
    "num_chains = len(X_list)\n",
    "\n",
    "\n",
    "## Divide in train val and test\n",
    "proportion_tr = 0.8\n",
    "proportion_val = 0.1\n",
    "proportion_tst = 1 -( proportion_val + proportion_tr)\n",
    "\n",
    "num_tr = 10000\n",
    "num_val = 5000\n",
    "num_tst = 5000\n",
    "\n",
    "train_X = [X_list[i] for i in range(num_tr)]\n",
    "train_Y = [Y_list[i] for i in range(num_tr)]\n",
    "\n",
    "val_X = [X_list[i] for i in range(num_tr, num_tr + num_val)]\n",
    "val_Y = [Y_list[i] for i in range(num_tr, num_tr + num_val)]\n",
    "\n",
    "tst_X = [X_list[i] for i in range(num_tr + num_val,  num_tr + num_val + num_tst)]\n",
    "tst_Y = [Y_list[i] for i in range(num_tr + num_val,  num_tr + num_val + num_tst)]\n",
    "\n",
    "# Create the objects with the hyperparameters that will be fed to the network\n",
    "train_config = get_config()\n",
    "eval_config = get_config( )\n",
    "\n",
    "###### Over Set parameters #####\n",
    "train_config.X_dim  = X_dim\n",
    "eval_config.X_dim  = X_dim\n",
    "train_config.num_steps  = num_steps\n",
    "eval_config.num_steps  = num_steps\n",
    "\n",
    "train_config.vocab_size = 2\n",
    "eval_config.vocab_size= 2\n",
    "\n",
    "\n",
    "eval_config.batch_size = 2\n",
    "    \n",
    "#eval_config.num_steps = 1\n",
    "\n",
    "print (\"Number of total initial chains %i\"%len(X_list))\n",
    "print (\"Dimensionality of chains (num_step,X_dim)\",X_list[0].shape )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Summary name KL Loss is illegal; using KL_Loss instead.\n",
      "INFO:tensorflow:Summary name Total Loss is illegal; using Total_Loss instead.\n",
      "Creating Validation model\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#subprocess.Popen([\"tensorboard\",\"--logdir=tensorboard\"])\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    initializer = tf.random_uniform_initializer(-train_config.init_scale,\n",
    "                                                train_config.init_scale)\n",
    "\n",
    "    with tf.name_scope(\"Train\"):\n",
    "        train_input = BBB_LSTM_Artificial_Data_Input(batch_size = train_config.batch_size, \n",
    "                                                        X = train_X, Y = train_Y,  name=\"TrainInput\")\n",
    "        \n",
    "        with tf.variable_scope(\"Model\", reuse=None, initializer=initializer):\n",
    "            m = PTBModel(is_training=True, config=train_config, input_=train_input)\n",
    "        tf.summary.scalar(\"Training_Loss\", m.cost)\n",
    "        tf.summary.scalar(\"Learning_Rate\", m.lr)\n",
    "        tf.summary.scalar(\"KL Loss\", m.kl_loss)\n",
    "        tf.summary.scalar(\"Total Loss\", m.total_loss)\n",
    "\n",
    "    print (\"Creating Validation model\")\n",
    "    with tf.name_scope(\"Valid\"):\n",
    "        valid_input = BBB_LSTM_Artificial_Data_Input(batch_size = eval_config.batch_size, \n",
    "                                                            X = val_X, Y = val_Y,  name=\"ValidInput\")\n",
    "        \n",
    "        with tf.variable_scope(\"Model\", reuse=True, initializer=initializer):\n",
    "            mvalid = PTBModel(is_training=False, config=eval_config, input_=valid_input)\n",
    "        tf.summary.scalar(\"Validation_Loss\", mvalid.cost)\n",
    "\n",
    "    with tf.name_scope(\"Test\"):\n",
    "        test_input = BBB_LSTM_Artificial_Data_Input(batch_size = eval_config.batch_size, \n",
    "                                                            X = tst_X, Y = tst_Y,  name=\"TestInput\")\n",
    "            \n",
    "        with tf.variable_scope(\"Model\", reuse=True, initializer=initializer):\n",
    "            mtest = PTBModel(is_training=False, config=eval_config,\n",
    "                             input_=test_input)\n",
    "\n",
    "    models = {\"Train\": m, \"Valid\": mvalid, \"Test\": mtest}\n",
    "    for name, model in models.items():\n",
    "        model.export_ops(name)\n",
    "    metagraph = tf.train.export_meta_graph()\n",
    "    soft_placement = False\n",
    "    if global_num_gpus > 1:\n",
    "        soft_placement = True\n",
    "        util.auto_parallel(metagraph, m)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./saved_model/model.ckpt-58220\n",
      "INFO:tensorflow:Starting standard services.\n",
      "INFO:tensorflow:Saving checkpoint to path ./saved_model/model.ckpt\n",
      "INFO:tensorflow:Starting queue runners.\n",
      "INFO:tensorflow:Recording summary at step 58220.\n",
      "Epoch: 1 Learning rate: 0.500\n",
      "0.000 perplexity: 1.975 speed: 37 wps\n",
      "KL is 0.41412240266799927\n",
      "0.001 perplexity: 2.012 speed: 356 wps\n",
      "KL is 0.4121624827384949\n",
      "0.101 perplexity: 1.987 speed: 4795 wps\n",
      "KL is 0.4344922602176666\n",
      "0.201 perplexity: 1.990 speed: 5710 wps\n",
      "KL is 0.4457932412624359\n",
      "0.301 perplexity: 1.995 speed: 6118 wps\n",
      "KL is 0.45759135484695435\n",
      "0.401 perplexity: 1.995 speed: 6335 wps\n",
      "KL is 0.4566430151462555\n",
      "0.501 perplexity: 1.994 speed: 6498 wps\n",
      "KL is 0.45681506395339966\n",
      "0.601 perplexity: 1.995 speed: 6607 wps\n",
      "KL is 0.4593292772769928\n",
      "0.701 perplexity: 1.997 speed: 6695 wps\n",
      "KL is 0.4530620574951172\n",
      "0.801 perplexity: 1.997 speed: 6763 wps\n",
      "KL is 0.4470798373222351\n",
      "0.901 perplexity: 1.997 speed: 6814 wps\n",
      "KL is 0.44141408801078796\n",
      "Epoch: 1 Train Perplexity: 1.997\n",
      "Epoch: 1 Valid Perplexity: 2.007\n",
      "Epoch: 2 Learning rate: 0.450\n",
      "0.000 perplexity: 1.892 speed: 2757 wps\n",
      "KL is 0.43438440561294556\n",
      "0.001 perplexity: 2.030 speed: 4078 wps\n",
      "KL is 0.4393104314804077\n",
      "0.101 perplexity: 1.996 speed: 5631 wps\n",
      "KL is 0.43655288219451904\n",
      "0.201 perplexity: 1.999 speed: 6262 wps\n",
      "KL is 0.4250113070011139\n",
      "0.301 perplexity: 2.000 speed: 6184 wps\n",
      "KL is 0.4284875988960266\n",
      "0.401 perplexity: 1.998 speed: 6042 wps\n",
      "KL is 0.4301774799823761\n",
      "0.501 perplexity: 1.997 speed: 6157 wps\n",
      "KL is 0.424103319644928\n",
      "0.601 perplexity: 1.994 speed: 6203 wps\n",
      "KL is 0.41908466815948486\n",
      "0.701 perplexity: 1.995 speed: 5955 wps\n",
      "KL is 0.41250067949295044\n",
      "0.801 perplexity: 1.994 speed: 5852 wps\n",
      "KL is 0.42460981011390686\n",
      "0.901 perplexity: 1.994 speed: 5830 wps\n",
      "KL is 0.42519012093544006\n",
      "Epoch: 2 Train Perplexity: 1.993\n",
      "Epoch: 2 Valid Perplexity: 2.016\n",
      "Epoch: 3 Learning rate: 0.405\n",
      "0.000 perplexity: 1.766 speed: 4747 wps\n",
      "KL is 0.4114842116832733\n",
      "0.001 perplexity: 2.053 speed: 6356 wps\n",
      "KL is 0.41160160303115845\n",
      "0.101 perplexity: 1.982 speed: 7242 wps\n",
      "KL is 0.4137822687625885\n",
      "0.201 perplexity: 1.986 speed: 7183 wps\n",
      "KL is 0.4061681628227234\n",
      "0.301 perplexity: 1.991 speed: 7200 wps\n",
      "KL is 0.3994753658771515\n",
      "0.401 perplexity: 1.991 speed: 7202 wps\n",
      "KL is 0.4001210629940033\n",
      "0.501 perplexity: 1.990 speed: 6863 wps\n",
      "KL is 0.40336745977401733\n",
      "0.601 perplexity: 1.989 speed: 6426 wps\n",
      "KL is 0.3988627791404724\n",
      "0.701 perplexity: 1.990 speed: 5780 wps\n",
      "KL is 0.3934714198112488\n",
      "0.801 perplexity: 1.989 speed: 5575 wps\n",
      "KL is 0.39052462577819824\n",
      "0.901 perplexity: 1.988 speed: 5492 wps\n",
      "KL is 0.3901631832122803\n",
      "Epoch: 3 Train Perplexity: 1.987\n",
      "Epoch: 3 Valid Perplexity: 1.994\n",
      "Epoch: 4 Learning rate: 0.364\n",
      "0.000 perplexity: 1.750 speed: 4615 wps\n",
      "KL is 0.39801689982414246\n",
      "0.001 perplexity: 2.064 speed: 5095 wps\n",
      "KL is 0.3956936299800873\n",
      "0.101 perplexity: 1.979 speed: 4911 wps\n",
      "KL is 0.3855619430541992\n",
      "0.201 perplexity: 1.979 speed: 5700 wps\n",
      "KL is 0.38325101137161255\n",
      "INFO:tensorflow:Recording summary at step 90308.\n",
      "0.301 perplexity: 1.981 speed: 5905 wps\n",
      "KL is 0.3904085159301758\n",
      "0.401 perplexity: 1.981 speed: 6140 wps\n",
      "KL is 0.38613027334213257\n",
      "0.501 perplexity: 1.980 speed: 6274 wps\n",
      "KL is 0.38746002316474915\n",
      "0.601 perplexity: 1.979 speed: 6047 wps\n",
      "KL is 0.38113799691200256\n",
      "0.701 perplexity: 1.980 speed: 5683 wps\n",
      "KL is 0.3763352632522583\n",
      "0.801 perplexity: 1.979 speed: 5677 wps\n",
      "KL is 0.3719336688518524\n",
      "0.901 perplexity: 1.978 speed: 5809 wps\n",
      "KL is 0.37353265285491943\n",
      "Epoch: 4 Train Perplexity: 1.978\n",
      "Epoch: 4 Valid Perplexity: 1.989\n",
      "Epoch: 5 Learning rate: 0.328\n",
      "0.000 perplexity: 1.961 speed: 4326 wps\n",
      "KL is 0.3818979561328888\n",
      "0.001 perplexity: 2.026 speed: 3406 wps\n",
      "KL is 0.3681236207485199\n",
      "0.101 perplexity: 1.972 speed: 6398 wps\n",
      "KL is 0.3781907558441162\n",
      "0.201 perplexity: 1.975 speed: 6393 wps\n",
      "KL is 0.3675310015678406\n",
      "0.301 perplexity: 1.975 speed: 5945 wps\n",
      "KL is 0.37438061833381653\n",
      "0.401 perplexity: 1.977 speed: 5819 wps\n",
      "KL is 0.36469465494155884\n",
      "0.501 perplexity: 1.976 speed: 5925 wps\n",
      "KL is 0.3607373833656311\n",
      "0.601 perplexity: 1.975 speed: 5909 wps\n",
      "KL is 0.37181538343429565\n",
      "0.701 perplexity: 1.977 speed: 5323 wps\n",
      "KL is 0.3631916046142578\n",
      "0.801 perplexity: 1.976 speed: 5320 wps\n",
      "KL is 0.35932227969169617\n",
      "0.901 perplexity: 1.976 speed: 5423 wps\n",
      "KL is 0.35845649242401123\n",
      "Epoch: 5 Train Perplexity: 1.975\n",
      "Epoch: 5 Valid Perplexity: 2.000\n",
      "Epoch: 6 Learning rate: 0.295\n",
      "0.000 perplexity: 1.787 speed: 4175 wps\n",
      "KL is 0.3610402047634125\n",
      "0.001 perplexity: 1.974 speed: 6550 wps\n",
      "KL is 0.356636643409729\n",
      "0.101 perplexity: 1.966 speed: 7327 wps\n",
      "KL is 0.3516588509082794\n",
      "0.201 perplexity: 1.971 speed: 7307 wps\n",
      "KL is 0.3506464958190918\n",
      "0.301 perplexity: 1.971 speed: 7293 wps\n",
      "KL is 0.35640957951545715\n",
      "0.401 perplexity: 1.971 speed: 6716 wps\n",
      "KL is 0.3480237126350403\n",
      "0.501 perplexity: 1.969 speed: 6543 wps\n",
      "KL is 0.35547685623168945\n",
      "0.601 perplexity: 1.968 speed: 6552 wps\n",
      "KL is 0.35192278027534485\n",
      "0.701 perplexity: 1.969 speed: 6617 wps\n",
      "KL is 0.3476436734199524\n",
      "0.801 perplexity: 1.968 speed: 6510 wps\n",
      "KL is 0.35229164361953735\n",
      "0.901 perplexity: 1.968 speed: 6349 wps\n",
      "KL is 0.3499187231063843\n",
      "Epoch: 6 Train Perplexity: 1.967\n",
      "Epoch: 6 Valid Perplexity: 1.989\n",
      "Epoch: 7 Learning rate: 0.266\n",
      "0.000 perplexity: 1.775 speed: 5009 wps\n",
      "KL is 0.3533218801021576\n",
      "0.001 perplexity: 2.075 speed: 6326 wps\n",
      "KL is 0.3596041798591614\n",
      "0.101 perplexity: 1.968 speed: 7047 wps\n",
      "KL is 0.3523704707622528\n",
      "0.201 perplexity: 1.965 speed: 6909 wps\n",
      "KL is 0.3489609658718109\n",
      "0.301 perplexity: 1.967 speed: 7030 wps\n",
      "KL is 0.35109224915504456\n",
      "0.401 perplexity: 1.967 speed: 7068 wps\n",
      "KL is 0.34690138697624207\n",
      "INFO:tensorflow:Recording summary at step 122983.\n",
      "0.501 perplexity: 1.965 speed: 6981 wps\n",
      "KL is 0.3434484302997589\n",
      "0.601 perplexity: 1.964 speed: 6987 wps\n",
      "KL is 0.3483601212501526\n",
      "0.701 perplexity: 1.966 speed: 7001 wps\n",
      "KL is 0.33581218123435974\n",
      "0.801 perplexity: 1.966 speed: 6973 wps\n",
      "KL is 0.34205201268196106\n",
      "0.901 perplexity: 1.965 speed: 6990 wps\n",
      "KL is 0.33884114027023315\n",
      "Epoch: 7 Train Perplexity: 1.964\n",
      "Epoch: 7 Valid Perplexity: 2.000\n",
      "Epoch: 8 Learning rate: 0.239\n",
      "0.000 perplexity: 2.088 speed: 4677 wps\n",
      "KL is 0.33693215250968933\n",
      "0.001 perplexity: 2.050 speed: 6175 wps\n",
      "KL is 0.33619529008865356\n",
      "0.101 perplexity: 1.959 speed: 6318 wps\n",
      "KL is 0.3320449888706207\n",
      "0.201 perplexity: 1.962 speed: 6185 wps\n",
      "KL is 0.3400570750236511\n",
      "0.301 perplexity: 1.962 speed: 6248 wps\n",
      "KL is 0.3420393168926239\n",
      "0.401 perplexity: 1.963 speed: 6116 wps\n",
      "KL is 0.3440941870212555\n",
      "0.501 perplexity: 1.961 speed: 6199 wps\n",
      "KL is 0.3359184265136719\n",
      "0.601 perplexity: 1.961 speed: 6143 wps\n",
      "KL is 0.33743494749069214\n",
      "0.701 perplexity: 1.963 speed: 5970 wps\n",
      "KL is 0.3328194320201874\n",
      "0.801 perplexity: 1.962 speed: 6000 wps\n",
      "KL is 0.3323502540588379\n",
      "0.901 perplexity: 1.961 speed: 6031 wps\n",
      "KL is 0.3391993045806885\n",
      "Epoch: 8 Train Perplexity: 1.961\n",
      "Epoch: 8 Valid Perplexity: 2.051\n",
      "Epoch: 9 Learning rate: 0.215\n",
      "0.000 perplexity: 1.910 speed: 5251 wps\n",
      "KL is 0.33323201537132263\n",
      "0.001 perplexity: 2.108 speed: 6674 wps\n",
      "KL is 0.3377543091773987\n",
      "0.101 perplexity: 1.954 speed: 5558 wps\n",
      "KL is 0.3342123329639435\n",
      "0.201 perplexity: 1.956 speed: 5800 wps\n",
      "KL is 0.32207295298576355\n",
      "0.301 perplexity: 1.957 speed: 6026 wps\n",
      "KL is 0.3298639953136444\n",
      "0.401 perplexity: 1.956 speed: 6204 wps\n",
      "KL is 0.3324529826641083\n",
      "0.501 perplexity: 1.955 speed: 6343 wps\n",
      "KL is 0.33601120114326477\n",
      "0.601 perplexity: 1.955 speed: 6426 wps\n",
      "KL is 0.3410068452358246\n",
      "0.701 perplexity: 1.956 speed: 6488 wps\n",
      "KL is 0.33157074451446533\n",
      "0.801 perplexity: 1.956 speed: 6500 wps\n",
      "KL is 0.33082205057144165\n",
      "0.901 perplexity: 1.955 speed: 6442 wps\n",
      "KL is 0.3329068422317505\n",
      "Epoch: 9 Train Perplexity: 1.955\n",
      "Epoch: 9 Valid Perplexity: 2.004\n",
      "Epoch: 10 Learning rate: 0.194\n",
      "0.000 perplexity: 1.932 speed: 4949 wps\n",
      "KL is 0.32645606994628906\n",
      "0.001 perplexity: 2.035 speed: 6186 wps\n",
      "KL is 0.33247148990631104\n",
      "0.101 perplexity: 1.948 speed: 4531 wps\n",
      "KL is 0.32883498072624207\n",
      "0.201 perplexity: 1.953 speed: 4833 wps\n",
      "KL is 0.32505881786346436\n",
      "0.301 perplexity: 1.955 speed: 4945 wps\n",
      "KL is 0.33073821663856506\n",
      "0.401 perplexity: 1.955 speed: 4727 wps\n",
      "KL is 0.33973678946495056\n",
      "0.501 perplexity: 1.954 speed: 4888 wps\n",
      "KL is 0.3207443654537201\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.601 perplexity: 1.954 speed: 5137 wps\n",
      "KL is 0.3292502462863922\n",
      "0.701 perplexity: 1.955 speed: 5248 wps\n",
      "KL is 0.3180225193500519\n",
      "INFO:tensorflow:Recording summary at step 155923.\n",
      "0.801 perplexity: 1.954 speed: 5420 wps\n",
      "KL is 0.326506108045578\n",
      "0.901 perplexity: 1.953 speed: 5503 wps\n",
      "KL is 0.3240233361721039\n",
      "Epoch: 10 Train Perplexity: 1.953\n",
      "Epoch: 10 Valid Perplexity: 2.007\n",
      "Epoch: 11 Learning rate: 0.174\n",
      "0.000 perplexity: 1.706 speed: 4224 wps\n",
      "KL is 0.3213675320148468\n",
      "0.001 perplexity: 1.977 speed: 6165 wps\n",
      "KL is 0.3236992657184601\n",
      "0.101 perplexity: 1.948 speed: 6481 wps\n",
      "KL is 0.32956990599632263\n",
      "0.201 perplexity: 1.952 speed: 6060 wps\n",
      "KL is 0.3218826353549957\n",
      "0.301 perplexity: 1.953 speed: 5705 wps\n",
      "KL is 0.31885090470314026\n",
      "0.401 perplexity: 1.952 speed: 5730 wps\n",
      "KL is 0.3240964114665985\n",
      "0.501 perplexity: 1.951 speed: 5962 wps\n",
      "KL is 0.3265066146850586\n",
      "0.601 perplexity: 1.950 speed: 6136 wps\n",
      "KL is 0.32736149430274963\n",
      "0.701 perplexity: 1.952 speed: 6272 wps\n",
      "KL is 0.3160449266433716\n",
      "0.801 perplexity: 1.951 speed: 6370 wps\n",
      "KL is 0.3215078115463257\n",
      "0.901 perplexity: 1.950 speed: 6448 wps\n",
      "KL is 0.32344022393226624\n",
      "Epoch: 11 Train Perplexity: 1.950\n",
      "Epoch: 11 Valid Perplexity: 2.017\n",
      "Epoch: 12 Learning rate: 0.157\n",
      "0.000 perplexity: 1.548 speed: 4729 wps\n",
      "KL is 0.322386771440506\n",
      "0.001 perplexity: 1.993 speed: 6634 wps\n",
      "KL is 0.3168228566646576\n",
      "0.101 perplexity: 1.950 speed: 7200 wps\n",
      "KL is 0.3131800591945648\n",
      "0.201 perplexity: 1.952 speed: 7145 wps\n",
      "KL is 0.3091387450695038\n",
      "0.301 perplexity: 1.953 speed: 7155 wps\n",
      "KL is 0.31005942821502686\n",
      "0.401 perplexity: 1.952 speed: 5995 wps\n",
      "KL is 0.3186054825782776\n",
      "0.501 perplexity: 1.951 speed: 6059 wps\n",
      "KL is 0.31745854020118713\n",
      "0.601 perplexity: 1.950 speed: 6190 wps\n",
      "KL is 0.31578993797302246\n",
      "0.701 perplexity: 1.950 speed: 6315 wps\n",
      "KL is 0.3129391372203827\n",
      "0.801 perplexity: 1.950 speed: 6420 wps\n",
      "KL is 0.31261083483695984\n",
      "0.901 perplexity: 1.949 speed: 6495 wps\n",
      "KL is 0.3216325640678406\n",
      "Epoch: 12 Train Perplexity: 1.949\n",
      "Epoch: 12 Valid Perplexity: 2.021\n",
      "Epoch: 13 Learning rate: 0.141\n",
      "0.000 perplexity: 1.677 speed: 4746 wps\n",
      "KL is 0.31158536672592163\n",
      "0.001 perplexity: 1.956 speed: 6642 wps\n",
      "KL is 0.31239551305770874\n",
      "0.101 perplexity: 1.944 speed: 7223 wps\n",
      "KL is 0.31346428394317627\n",
      "0.201 perplexity: 1.947 speed: 7240 wps\n",
      "KL is 0.30816182494163513\n",
      "0.301 perplexity: 1.949 speed: 7242 wps\n",
      "KL is 0.30276209115982056\n",
      "0.401 perplexity: 1.948 speed: 7216 wps\n",
      "KL is 0.30623260140419006\n",
      "0.501 perplexity: 1.948 speed: 6584 wps\n",
      "KL is 0.30149412155151367\n",
      "0.601 perplexity: 1.946 speed: 5872 wps\n",
      "KL is 0.3061210513114929\n",
      "0.701 perplexity: 1.947 speed: 5492 wps\n",
      "KL is 0.30462634563446045\n",
      "0.801 perplexity: 1.947 speed: 4873 wps\n",
      "KL is 0.30669286847114563\n",
      "0.901 perplexity: 1.945 speed: 4950 wps\n",
      "KL is 0.3071615993976593\n",
      "INFO:tensorflow:Recording summary at step 187975.\n",
      "Epoch: 13 Train Perplexity: 1.945\n",
      "Epoch: 13 Valid Perplexity: 1.994\n",
      "Epoch: 14 Learning rate: 0.127\n",
      "0.000 perplexity: 2.051 speed: 3201 wps\n",
      "KL is 0.30518466234207153\n",
      "0.001 perplexity: 1.973 speed: 3371 wps\n",
      "KL is 0.30610179901123047\n",
      "0.101 perplexity: 1.939 speed: 4742 wps\n",
      "KL is 0.30910202860832214\n",
      "0.201 perplexity: 1.942 speed: 5539 wps\n",
      "KL is 0.30369922518730164\n",
      "0.301 perplexity: 1.944 speed: 5978 wps\n",
      "KL is 0.3029850423336029\n",
      "0.401 perplexity: 1.944 speed: 5500 wps\n",
      "KL is 0.31099164485931396\n",
      "0.501 perplexity: 1.944 speed: 5689 wps\n",
      "KL is 0.3011048436164856\n",
      "0.601 perplexity: 1.942 speed: 5864 wps\n",
      "KL is 0.30497056245803833\n",
      "0.701 perplexity: 1.943 speed: 6016 wps\n",
      "KL is 0.3044983446598053\n",
      "0.801 perplexity: 1.943 speed: 6142 wps\n",
      "KL is 0.29981184005737305\n",
      "0.901 perplexity: 1.942 speed: 6208 wps\n",
      "KL is 0.3035818934440613\n",
      "Epoch: 14 Train Perplexity: 1.942\n",
      "Epoch: 14 Valid Perplexity: 1.955\n",
      "Epoch: 15 Learning rate: 0.114\n",
      "0.000 perplexity: 2.166 speed: 5334 wps\n",
      "KL is 0.3114664554595947\n",
      "0.001 perplexity: 1.976 speed: 6429 wps\n",
      "KL is 0.30618441104888916\n",
      "0.101 perplexity: 1.943 speed: 6510 wps\n",
      "KL is 0.3024899959564209\n",
      "0.201 perplexity: 1.942 speed: 6763 wps\n",
      "KL is 0.3065589368343353\n",
      "0.301 perplexity: 1.943 speed: 6851 wps\n",
      "KL is 0.3055753707885742\n",
      "0.401 perplexity: 1.943 speed: 6788 wps\n",
      "KL is 0.3053150177001953\n",
      "0.501 perplexity: 1.942 speed: 6810 wps\n",
      "KL is 0.30734118819236755\n",
      "0.601 perplexity: 1.940 speed: 6862 wps\n",
      "KL is 0.309734046459198\n",
      "0.701 perplexity: 1.941 speed: 6897 wps\n",
      "KL is 0.3047790825366974\n",
      "0.801 perplexity: 1.942 speed: 6930 wps\n",
      "KL is 0.3055190443992615\n",
      "0.901 perplexity: 1.941 speed: 6874 wps\n",
      "KL is 0.30706486105918884\n",
      "Epoch: 15 Train Perplexity: 1.941\n",
      "Epoch: 15 Valid Perplexity: 1.976\n",
      "Epoch: 16 Learning rate: 0.103\n",
      "0.000 perplexity: 2.155 speed: 4999 wps\n",
      "KL is 0.3103579878807068\n",
      "0.001 perplexity: 1.973 speed: 6493 wps\n",
      "KL is 0.3041170537471771\n",
      "0.101 perplexity: 1.940 speed: 7037 wps\n",
      "KL is 0.30631595849990845\n",
      "0.201 perplexity: 1.940 speed: 7079 wps\n",
      "KL is 0.30148109793663025\n",
      "0.301 perplexity: 1.942 speed: 7140 wps\n",
      "KL is 0.3013175129890442\n",
      "0.401 perplexity: 1.943 speed: 7115 wps\n",
      "KL is 0.30276986956596375\n",
      "0.501 perplexity: 1.942 speed: 7122 wps\n",
      "KL is 0.3055301308631897\n",
      "0.601 perplexity: 1.941 speed: 7124 wps\n",
      "KL is 0.30087175965309143\n",
      "0.701 perplexity: 1.942 speed: 6936 wps\n",
      "KL is 0.3048596978187561\n",
      "0.801 perplexity: 1.943 speed: 6396 wps\n",
      "KL is 0.29397693276405334\n",
      "0.901 perplexity: 1.941 speed: 6071 wps\n",
      "KL is 0.307681679725647\n",
      "Epoch: 16 Train Perplexity: 1.941\n",
      "Epoch: 16 Valid Perplexity: 1.946\n",
      "Epoch: 17 Learning rate: 0.093\n",
      "0.000 perplexity: 2.152 speed: 2975 wps\n",
      "KL is 0.30213767290115356\n",
      "0.001 perplexity: 1.949 speed: 4990 wps\n",
      "KL is 0.29989007115364075\n",
      "0.101 perplexity: 1.936 speed: 5568 wps\n",
      "KL is 0.3032727539539337\n",
      "INFO:tensorflow:Saving checkpoint to path ./saved_model/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 219759.\n",
      "0.201 perplexity: 1.940 speed: 5244 wps\n",
      "KL is 0.3026614189147949\n",
      "0.301 perplexity: 1.941 speed: 5479 wps\n",
      "KL is 0.29930734634399414\n",
      "0.401 perplexity: 1.942 speed: 5605 wps\n",
      "KL is 0.2999894917011261\n",
      "0.501 perplexity: 1.940 speed: 5543 wps\n",
      "KL is 0.2969134449958801\n",
      "0.601 perplexity: 1.939 speed: 5553 wps\n",
      "KL is 0.2980116009712219\n",
      "0.701 perplexity: 1.940 speed: 5461 wps\n",
      "KL is 0.2940710484981537\n",
      "0.801 perplexity: 1.940 speed: 4985 wps\n",
      "KL is 0.30023401975631714\n",
      "0.901 perplexity: 1.938 speed: 4865 wps\n",
      "KL is 0.2985040843486786\n",
      "Epoch: 17 Train Perplexity: 1.939\n",
      "Epoch: 17 Valid Perplexity: 1.957\n",
      "Epoch: 18 Learning rate: 0.083\n",
      "0.000 perplexity: 1.697 speed: 4321 wps\n",
      "KL is 0.2965551018714905\n",
      "0.001 perplexity: 1.931 speed: 6336 wps\n",
      "KL is 0.2940332591533661\n",
      "0.101 perplexity: 1.934 speed: 5552 wps\n",
      "KL is 0.29395633935928345\n",
      "0.201 perplexity: 1.937 speed: 5990 wps\n",
      "KL is 0.2940292954444885\n",
      "0.301 perplexity: 1.939 speed: 5315 wps\n",
      "KL is 0.2992080748081207\n",
      "0.401 perplexity: 1.940 speed: 5482 wps\n",
      "KL is 0.2977086007595062\n",
      "0.501 perplexity: 1.938 speed: 5736 wps\n",
      "KL is 0.2902407646179199\n",
      "0.601 perplexity: 1.937 speed: 5936 wps\n",
      "KL is 0.29169097542762756\n",
      "0.701 perplexity: 1.939 speed: 6099 wps\n",
      "KL is 0.2925125062465668\n",
      "0.801 perplexity: 1.938 speed: 6224 wps\n",
      "KL is 0.2938094735145569\n",
      "0.901 perplexity: 1.937 speed: 6325 wps\n",
      "KL is 0.29846054315567017\n",
      "Epoch: 18 Train Perplexity: 1.937\n",
      "Epoch: 18 Valid Perplexity: 1.944\n",
      "Epoch: 19 Learning rate: 0.075\n",
      "0.000 perplexity: 1.713 speed: 5121 wps\n",
      "KL is 0.2993380129337311\n",
      "0.001 perplexity: 1.951 speed: 6611 wps\n",
      "KL is 0.29293665289878845\n",
      "0.101 perplexity: 1.938 speed: 7302 wps\n",
      "KL is 0.29460832476615906\n",
      "0.201 perplexity: 1.940 speed: 7309 wps\n",
      "KL is 0.28470757603645325\n",
      "0.301 perplexity: 1.942 speed: 7281 wps\n",
      "KL is 0.2953018546104431\n",
      "0.401 perplexity: 1.942 speed: 7278 wps\n",
      "KL is 0.28976938128471375\n",
      "0.501 perplexity: 1.940 speed: 7283 wps\n",
      "KL is 0.2932826280593872\n",
      "0.601 perplexity: 1.939 speed: 7271 wps\n",
      "KL is 0.29818615317344666\n",
      "0.701 perplexity: 1.939 speed: 6571 wps\n",
      "KL is 0.28365087509155273\n",
      "0.801 perplexity: 1.939 speed: 6591 wps\n",
      "KL is 0.2846655547618866\n",
      "0.901 perplexity: 1.938 speed: 6616 wps\n",
      "KL is 0.28976622223854065\n",
      "Epoch: 19 Train Perplexity: 1.938\n",
      "Epoch: 19 Valid Perplexity: 1.941\n",
      "Epoch: 20 Learning rate: 0.068\n",
      "0.000 perplexity: 1.769 speed: 5049 wps\n",
      "KL is 0.2795092761516571\n",
      "0.001 perplexity: 1.955 speed: 6584 wps\n",
      "KL is 0.29070404171943665\n",
      "0.101 perplexity: 1.930 speed: 6274 wps\n",
      "KL is 0.28508901596069336\n",
      "0.201 perplexity: 1.935 speed: 6648 wps\n",
      "KL is 0.2840372323989868\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.301 perplexity: 1.937 speed: 6835 wps\n",
      "KL is 0.28582289814949036\n",
      "INFO:tensorflow:Recording summary at step 252005.\n",
      "0.401 perplexity: 1.936 speed: 6942 wps\n",
      "KL is 0.2890756130218506\n",
      "0.501 perplexity: 1.935 speed: 7006 wps\n",
      "KL is 0.28720906376838684\n",
      "0.601 perplexity: 1.934 speed: 6718 wps\n",
      "KL is 0.2846355736255646\n",
      "0.701 perplexity: 1.936 speed: 6502 wps\n",
      "KL is 0.28831616044044495\n",
      "0.801 perplexity: 1.936 speed: 6550 wps\n",
      "KL is 0.2914477288722992\n",
      "0.901 perplexity: 1.935 speed: 6618 wps\n",
      "KL is 0.28681880235671997\n",
      "Epoch: 20 Train Perplexity: 1.935\n",
      "Epoch: 20 Valid Perplexity: 1.939\n",
      "Test Perplexity: 1.936\n",
      "Saving model to ./saved_model/.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "## Training !\n",
    "with tf.Graph().as_default():\n",
    "    tf.train.import_meta_graph(metagraph)\n",
    "    for model in models.values():\n",
    "        model.import_ops()\n",
    "    sv = tf.train.Supervisor(logdir=save_path)\n",
    "    config_proto = tf.ConfigProto(allow_soft_placement=soft_placement)\n",
    "    with sv.managed_session(config=config_proto) as session:\n",
    "\n",
    "        for i in range(train_config.max_max_epoch):\n",
    "            lr_decay = train_config.lr_decay ** max(i + 1 - train_config.max_epoch, 0.0)\n",
    "            m.assign_lr(session, train_config.learning_rate * lr_decay)\n",
    "\n",
    "            print(\"Epoch: %d Learning rate: %.3f\" % (i + 1, session.run(m.lr)))\n",
    "            train_perplexity = run_epoch(session, m, eval_op=m.train_op,\n",
    "                                         verbose=True)\n",
    "            print(\"Epoch: %d Train Perplexity: %.3f\" % (i + 1, train_perplexity))\n",
    "            valid_perplexity = run_epoch(session, mvalid)\n",
    "            print(\"Epoch: %d Valid Perplexity: %.3f\" % (i + 1, valid_perplexity))\n",
    "            \n",
    "        test_perplexity = run_epoch(session, mtest)\n",
    "        print(\"Test Perplexity: %.3f\" % test_perplexity)\n",
    "        \n",
    "\n",
    "        print(\"Saving model to %s.\" % save_path)\n",
    "        sv.saver.save(session, save_path, global_step=sv.global_step)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing\n",
      "INFO:tensorflow:Restoring parameters from ./saved_model/-258220\n",
      "INFO:tensorflow:Starting standard services.\n",
      "INFO:tensorflow:Saving checkpoint to path ./saved_model/model.ckpt\n",
      "INFO:tensorflow:Starting queue runners.\n",
      "INFO:tensorflow:Recording summary at step 258220.\n",
      "Test Perplexity: 1.936\n",
      "----------------------------------------------------------------\n",
      "------------------ Prediction of Output ---------------------\n",
      "Computing batch 0/2500\n",
      "Computing batch 1/2500\n",
      "Computing batch 2/2500\n",
      "Computing batch 3/2500\n",
      "Computing batch 4/2500\n",
      "Computing batch 5/2500\n",
      "Computing batch 6/2500\n",
      "Computing batch 7/2500\n",
      "Computing batch 8/2500\n",
      "Computing batch 9/2500\n",
      "Computing batch 10/2500\n",
      "Computing batch 11/2500\n",
      "Computing batch 12/2500\n",
      "Computing batch 13/2500\n",
      "Computing batch 14/2500\n",
      "Computing batch 15/2500\n",
      "Computing batch 16/2500\n",
      "Computing batch 17/2500\n",
      "Computing batch 18/2500\n",
      "Computing batch 19/2500\n",
      "Computing batch 20/2500\n",
      "Computing batch 21/2500\n",
      "Computing batch 22/2500\n",
      "Computing batch 23/2500\n",
      "Computing batch 24/2500\n",
      "Computing batch 25/2500\n",
      "Computing batch 26/2500\n",
      "Computing batch 27/2500\n",
      "Computing batch 28/2500\n",
      "Computing batch 29/2500\n",
      "Computing batch 30/2500\n",
      "Computing batch 31/2500\n",
      "Computing batch 32/2500\n",
      "Computing batch 33/2500\n",
      "Computing batch 34/2500\n",
      "Computing batch 35/2500\n",
      "Computing batch 36/2500\n",
      "Computing batch 37/2500\n",
      "Computing batch 38/2500\n",
      "Computing batch 39/2500\n",
      "Computing batch 40/2500\n",
      "Computing batch 41/2500\n",
      "Computing batch 42/2500\n",
      "Computing batch 43/2500\n",
      "Computing batch 44/2500\n",
      "Computing batch 45/2500\n",
      "Computing batch 46/2500\n",
      "Computing batch 47/2500\n",
      "Computing batch 48/2500\n",
      "Computing batch 49/2500\n",
      "Computing batch 50/2500\n",
      "Computing batch 51/2500\n",
      "Computing batch 52/2500\n",
      "Computing batch 53/2500\n",
      "Computing batch 54/2500\n",
      "Computing batch 55/2500\n",
      "Computing batch 56/2500\n",
      "Computing batch 57/2500\n",
      "Computing batch 58/2500\n",
      "Computing batch 59/2500\n",
      "Computing batch 60/2500\n",
      "Computing batch 61/2500\n",
      "Computing batch 62/2500\n",
      "Computing batch 63/2500\n",
      "Computing batch 64/2500\n",
      "Computing batch 65/2500\n",
      "Computing batch 66/2500\n",
      "Computing batch 67/2500\n",
      "Computing batch 68/2500\n",
      "Computing batch 69/2500\n",
      "Computing batch 70/2500\n",
      "Computing batch 71/2500\n",
      "Computing batch 72/2500\n",
      "Computing batch 73/2500\n",
      "Computing batch 74/2500\n",
      "Computing batch 75/2500\n",
      "Computing batch 76/2500\n",
      "Computing batch 77/2500\n",
      "Computing batch 78/2500\n",
      "Computing batch 79/2500\n",
      "Computing batch 80/2500\n",
      "Computing batch 81/2500\n",
      "Computing batch 82/2500\n",
      "Computing batch 83/2500\n",
      "Computing batch 84/2500\n",
      "Computing batch 85/2500\n",
      "Computing batch 86/2500\n",
      "Computing batch 87/2500\n",
      "Computing batch 88/2500\n",
      "Computing batch 89/2500\n",
      "Computing batch 90/2500\n",
      "Computing batch 91/2500\n",
      "Computing batch 92/2500\n",
      "Computing batch 93/2500\n",
      "Computing batch 94/2500\n",
      "Computing batch 95/2500\n",
      "Computing batch 96/2500\n",
      "Computing batch 97/2500\n",
      "Computing batch 98/2500\n",
      "Computing batch 99/2500\n",
      "Computing batch 100/2500\n"
     ]
    }
   ],
   "source": [
    "## Testing\n",
    "print (\"Testing\")\n",
    "predicted = []   # Variable to store predictions\n",
    "with tf.Graph().as_default():\n",
    "    tf.train.import_meta_graph(metagraph)\n",
    "    for model in models.values():\n",
    "        model.import_ops()\n",
    "    sv = tf.train.Supervisor(logdir=save_path)\n",
    "    config_proto = tf.ConfigProto(allow_soft_placement=soft_placement)\n",
    "    with sv.managed_session(config=config_proto) as session:\n",
    "        \n",
    "       # session = tf.Session()\n",
    "    \n",
    "        test_perplexity = run_epoch(session, mtest)\n",
    "        print(\"Test Perplexity: %.3f\" % test_perplexity)\n",
    "\n",
    "        print (\"----------------------------------------------------------------\")\n",
    "        print (\"------------------ Prediction of Output ---------------------\")\n",
    "\n",
    "       #  inputs, predicted = fetch_output(session, mtest)\n",
    "\n",
    "        costs = 0.0\n",
    "        state = session.run(model.initial_state)\n",
    "\n",
    "        inputs = []\n",
    "        outputs = []\n",
    "        targets = []\n",
    "        fetches = {\n",
    "            \"final_state\": model.final_state,\n",
    "            \"output\": model.output,\n",
    "            \"input\": model.input_data,\n",
    "            \"targets\": model.targets\n",
    "        }\n",
    "\n",
    "        for step in range(model.input.epoch_size):\n",
    "            feed_dict = {}\n",
    "            for i, (c, h) in enumerate(model.initial_state):\n",
    "                feed_dict[c] = state[i].c\n",
    "                feed_dict[h] = state[i].h\n",
    "\n",
    "            print (\"Computing batch %i/%i\"%(step, model.input.epoch_size))\n",
    "            vals = session.run(fetches, feed_dict)\n",
    "            state = vals[\"final_state\"]\n",
    "            output = vals[\"output\"]\n",
    "            input_i = vals[\"input\"]\n",
    "            \n",
    "            outputs.append(output)\n",
    "            inputs.append(input_i)\n",
    "            targets.append(vals[\"targets\"])\n",
    "            if (step == 100):\n",
    "                break;\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input and output of the first chain of the first batch\n",
      "[[ 0.02724501]\n",
      " [ 0.15398516]\n",
      " [-0.24624881]\n",
      " [ 0.01099645]\n",
      " [ 0.28329131]\n",
      " [ 0.88788235]\n",
      " [ 0.00593538]\n",
      " [-0.48042613]\n",
      " [-0.12275776]\n",
      " [ 0.77907795]\n",
      " [-0.64286244]\n",
      " [ 0.0563636 ]\n",
      " [-0.53884447]\n",
      " [-0.13336417]\n",
      " [-0.85664529]\n",
      " [-0.54134476]\n",
      " [ 0.25123903]\n",
      " [-0.41768441]\n",
      " [-0.06790691]\n",
      " [-0.12501006]]\n",
      "Soft Outputs\n",
      "[[ 0.45725206  0.54274791]\n",
      " [ 0.43493366  0.56506628]\n",
      " [ 0.50050414  0.49949589]\n",
      " [ 0.5203132   0.47968677]\n",
      " [ 0.53306264  0.46693736]\n",
      " [ 0.39954019  0.60045981]\n",
      " [ 0.56649953  0.43350056]\n",
      " [ 0.57772619  0.42227381]\n",
      " [ 0.5509553   0.44904479]\n",
      " [ 0.31849742  0.68150258]\n",
      " [ 0.47819853  0.52180141]\n",
      " [ 0.38064894  0.61935109]\n",
      " [ 0.41107842  0.58892161]\n",
      " [ 0.41290691  0.58709311]\n",
      " [ 0.46434805  0.53565192]\n",
      " [ 0.40163288  0.59836709]\n",
      " [ 0.31982931  0.68017071]\n",
      " [ 0.4070757   0.59292436]\n",
      " [ 0.41410673  0.58589327]\n",
      " [ 0.45205021  0.54794979]]\n",
      "Hard Outputs\n",
      "[1 1 0 0 0 1 0 0 0 1 1 1 1 1 1 1 1 1 1 1]\n",
      "targets\n",
      "[0 0 0 0 1 1 1 0 0 0 0 0 0 1 1 1 1 1 0 0]\n",
      "Prob of -1\n",
      "[ 0.02724501  0.15398516 -0.24624881  0.01099645  0.28329131  0.88788235\n",
      "  0.00593538 -0.48042613 -0.12275776  0.77907795 -0.64286244  0.0563636\n",
      " -0.53884447 -0.13336417 -0.85664529 -0.54134476  0.25123903 -0.41768441\n",
      " -0.06790691 -0.12501006]\n"
     ]
    }
   ],
   "source": [
    "print (\"Input and output of the first chain of the first batch\")\n",
    "print (inputs[0][0])\n",
    "\n",
    "print(\"Soft Outputs\")\n",
    "print (outputs[0][0])\n",
    "print(\"Hard Outputs\")\n",
    "selected_words = np.argmax(outputs[0][0], axis = 1)\n",
    "print (selected_words)\n",
    "print(\"targets\")\n",
    "print (targets[0][0])\n",
    "\n",
    "print (\"Prob of -1\")\n",
    "print (np.array(inputs[0][0])[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/montoya/anaconda3/lib/python3.6/site-packages/matplotlib/axes/_axes.py:545: UserWarning: No labelled objects found. Use label='...' kwarg on individual plots.\n",
      "  warnings.warn(\"No labelled objects found. \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEJCAYAAACKWmBmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VFX6wPHvmVQ6AUKAABKaGCGIIE0RpSjYwFVX7HWx\nr6vrrg3L2tbfWtay9gZWVFBBBBGBgIiFIj2U0EMILRBISJvM+f0xJZMwM5ly75Twfp4nD1Puvefl\nJpk3pyutNUIIIYQ3lkgHIIQQIrpJohBCCOGTJAohhBA+SaIQQgjhkyQKIYQQPkmiEEII4ZMkCiGE\nED5JohBCCOGTJAohhBA+xUc6AF9atWqlO3XqFNS5JSUlNGrUyNiATBALcUqMxomFOCVG40R7nMuW\nLduvtU6t80CtddR+9e3bVwdr/vz5QZ8bTrEQp8RonFiIU2I0TrTHCSzVfnwWS9OTEEIInyRRCCGE\n8EkShRBCCJ8MSRRKqVFKqQ1KqVyl1AMe3m+mlPpWKbVSKbVWKXWDEeUKIYQwX8iJQikVB7wGjAYy\ngSuUUpm1DrsDWKe17g2cBbyglEoMtWwhhBDmM6JG0R/I1Vpv0VpXAJOBMbWO0UATpZQCGgOFgNWA\nsoUQQpjMiESRDux0e57neM3d/4CTgHxgNXC31tpmQNlCCCFMpnSIW6EqpS4FRmmtb3Y8vwYYoLW+\ns9YxpwP3Al2AOUBvrfVhD9cbD4wHSEtL6zt58uSg4iouLqZx48ZBnRtOsRCnxGicWIgz2mPUWnPo\nUBEpKc0jHUqdov1enn322cu01v3qPNCfyRa+voBBwGy35w8CD9Y65jtgiNvzeUD/uq4tE+6ig8Ro\nnFiIM9pjnPDIqzo+sY9et25zpEOpU7TfS8I44W4J0E0pleHooB4HTK91zA5gOIBSKg04EdhiQNlC\niONIZWUlr7/xBVZrFQsXLot0OMeNkNd60lpblVJ3ArOBOOB9rfVapdStjvffBJ4EJiqlVgMKuF9r\nvT/UsoUQx5f585dQWFgEwI6dBRGO5vhhyKKAWuuZwMxar73p9jgfOMeIsoQQx68pU390Pd4piSJs\nZGa2ECImWK1Wvv5mnuu51CjCRxKFECImLFiwjP37D9K0qX0U0Y4duyMc0fFDEoUQIiZ8OeUHAG6+\n6WIA8vL2UFVVFcmQjhuSKIQQUa+qqoqvvp4LwLXXXEjz5o2prLSyZ8+BCEd2fJBEIYSIegsXLmPf\nvoN07dqRrKzutG6dAsCOHdJPEQ6SKIQQUc852umyS0eilKJ1qn1WtvRThIckCiFEVKuqqmLqV/ZE\nceklIwGqaxQy8iksJFEIIaLazz+vYM+eA3Tu3J4+fXoAkOZIFDKXIjwkUQghoppztNOll4zAvlMB\ntE5z9lFI01M4SKIQQkQtm83G1K/so50uu7R6cQdpegovSRRCiKi1ePEKdu/eR6dO7ejbt3rjzDQZ\n9RRWkiiEEFHryylzAHsntrPZCSAlpQkJCfHs33+Qo0dLIxXecUMShRAiKtmbnaqHxbqzWCx06NAG\ngJ0794Q9tuONJAohRFT69ddV7Nq1l44d23LaaT2Ped+ZKKRD23ySKIQQUWnKVGez04gazU5OHSVR\nhI0kCiFE1LHZbK7Z2M5JdrV17NgWgJ150vRkNkkUQoios2TJGnbuLKB9+zQGDOjl8RipUYSPJAoh\nRNRxjna65E8jsFg8f0w5axQyl8J8hiQKpdQopdQGpVSuUuoBL8ecpZRaoZRaq5RaYES5Qoj6R2td\nYxFAbzp2lBpFuIS8Z7ZSKg54DRgJ5AFLlFLTtdbr3I5pDrwOjNJa71BKtQ61XCFE/bR06Vq2b8+n\nXbtUBg3q7fW46lFPBWitPXZ4C2MYUaPoD+RqrbdorSuAycCYWsdcCXyltd4BoLXea0C5Qoh6yDna\nyVezE0CTJo1ISWlKeXkF+/YVhiu845IRiSId2On2PM/xmrvuQIpSKlsptUwpda0B5Qoh6hmtdY3Z\n2HVxr1UI84Tc9BRAOX2B4UAD4Bel1K9a6421D1RKjQfGA6SlpZGdnR1UgcXFxUGfG06xEKfEaJxY\niDOSMW7cuJOtW3fRokVTKiuLvMbhjLFRwwQAZs76keLifWGM1D+x8P32hxGJYhfQwe15e8dr7vKA\nA1rrEqBEKbUQ6A0ckyi01m8DbwP069dPn3XWWUEFlZ2dTbDnhlMsxCkxGicW4oxkjLN/eBmAcZeP\nZvjwYV6Pc8bYp89ifvl1Lc2atYrK+xoL329/GNH0tAToppTKUEolAuOA6bWOmQacoZSKV0o1BAYA\nOQaULYSoJ/wd7eRORj6FR8g1Cq21VSl1JzAbiAPe11qvVUrd6nj/Ta11jlLqe2AVYAPe1VqvCbVs\nIUT9sXLlBnJzd9C6dQuGDDnVr3M6dnDMpZA+ClMZ0kehtZ4JzKz12pu1nj8HPGdEeUKI+sdZm/jT\nxcOJi4vz6xxXjWKn1CjMJDOzhRARZx/t5Nzy1L9mJ3CbnS01ClNJohBCRNyaNbls3LidVq1SGDq0\nr9/ntW3bCovFQkHBfsrLK0yM8PgmiUIIEXHO2sTFY4cRH+9/i3h8fDzp6faFHvJkFVnTSKIQQkRc\noKOd3Dn7KXbK4oCmkUQhhIiotWtzycnZQosWzTjrrH4Bny8jn8wniUIIEVHO2sTFY4eRkJAQ8Pky\n8sl8kiiEEBEVzGgndzLyyXySKIQQEZOTs4W1azeTktKU4cP7B3UN2enOfJIohBAR41xSfMxFZwXV\n7ASy0104SKIQQkRM9Winc4K+Rge3GoXW2pC4RE2SKIQQEbFx4zZWrdpIs2ZNGDFiYNDXad68CY0b\nN6SkpJSDBw8bGKFwkkQhhIgIZ21izEVnkZgYXLMTgFJK5lKYTBKFECIiqkc7jQj5WtVzKaRD2wyS\nKIQQYZebu4MVKzbQpEkjzjlncMjXq55LITUKM0iiECJG5OXtYcmS+rGNi3O000UXDiUpKTHk61XP\npZAahRkkUQgRI8Zd+U8GDr6GNWs2RTqUkH05xZ4oQhnt5K56LoXUKMwgiUKIGFBRUcnvv6/BZrMx\n9au5kQ4nJFu25LF8eQ6NGzfknHMGGXJN1xBZaXoyhSQKIfxw4MAhtm7Ni1j5GzZso7LSCsC06fMj\nFocRnM1OF14wlAYNkg25pjQ9mcuQRKGUGqWU2qCUylVKPeDjuNOUUlal1KVGlCtEuJwz6lZ6Zl1C\nYWFRRMpftWqj6/Eff6xn+/b8iMRhBGeiMGK0k1N6emuUUuTn78NqtRp2XbMcPVrKKaf+mVGjb8Nm\ns0U6nDqFnCiUUnHAa8BoIBO4QimV6eW4/wN+CLVMIcLJZrOxevUmjh4tY/nynIjEsGr1xhrPp3+b\nHZE4QrV9ez5LlqylUaMGjB59hmHXTUpKpE2bVthsNvLz9xl2XbO8/c5UVq7cwOwfFjPVMZ8kmhlR\no+gP5Gqtt2itK4DJwBgPx90FTAX2GlCmEGFz4MAhV7PPipUbIhLDqlX2DuyLxw4DYNr07IjEEarZ\nsxcDcM7IQYY1Ozm5hshGefNTeXkFzz0/0fX8kcdei/pakBGJIh3Y6fY8z/Gai1IqHbgYeMOA8oQI\nK/e/UFdGKFGsdox0evCBm4iLi2PBgmUcOhR7y1X8OPc3wJ4ojBYrGxhNnDiN/Px99OrVjc6d27Nh\nwzY++mhGpMPyyf/NaUPzEnC/1tqmlPJ5oFJqPDAeIC0tjezs7KAKLC4uDvrccIqFOI/3GH/9ba3r\n8c+Ll4dUTjBxFhWVsGvXXpKTEzlyZC9ZvTrzx4pNPP/C24wYHviOcGbE6A+bzcbs2YsAaNJEGX4f\nlaoEIHvBz7Rr1zDoaxupdpxWaxWPP/E6AH+6+HQqK6t45t8f8eBDL5Ge3pjExHB9JAdIax3SFzAI\nmO32/EHgwVrHbAW2Ob6KsTc/ja3r2n379tXBmj9/ftDnhlMsxHm8x/jOO1M1liyNJUvHJ56qS0vL\ngr5WMHHOn/+7xpKlBwy6Smut9Usvf6yxZOk/X35f0HH4Lm++KdddunStxpKlT8gYpW02W0jX8hTj\ny6/Y78tttz8V0rWNVDvOiROnaSxZ+sSTLtJWq1VbrVbdM+tPGkuWfvmVj8MeH7BU+/E5b0TT0xKg\nm1IqQymVCIwDptdKRhla605a607AFOB2rfU3BpQthOl25Vd3q1mtVtat2xLW8p0d2Vm9ugP2RfQA\nZs5aRHl5RVhjCcWPc38FYOSIgdTVshCMDu2ju4+iqqqKZ559F4CHHriZuLg44uLieOqJOwF4+pl3\nKS4+GskQvQo5UWitrcCdwGwgB/hCa71WKXWrUurWUK8vRKTlOxKF88Mt3P0Uzo7srKxuAHTqlE5W\nVneKi48yf/6SsMYSijk/2hPFiOEDTLl+tG9gNGXKHDZu3E5GRjpXXDHK9fpFF51F//492bu3kFde\n/TSCEXpnyDwKrfVMrXV3rXUXrfXTjtfe1Fq/6eHY67XWU4woV4hwcHZm9+/fE4CVqzb6OtxwtWsU\nUF2riJXJd6WlZSxa9AcAw01LFNG71LjNZuPpf9trEw/888Yau/kppXjmqb8C8J/nJkblnhoyM1uI\nOjibns5zjPsP5xDZqqoq1qzZDECvXt1cr4+56GzAPp8iFiZs/fzzCsrLK+jTpwetWqWYUkarVikk\nJydx6NARDh8uNqWMYM2YsZDVqzeRnt6a66676Jj3hw8fwPDhAygqOlJj6Gy0kEQhRB2cNYrRo+yJ\nYuXKDWHbcnPz5p2UlpbRoUMbUlKaul4/9dSTaN8+jfz8fSxbti4ssYRizo+/ADBiePA72dUlWjcw\n0lrz1DNvA/DPf1zvdbXcp5+091W8/MonFBTsD1t8/pBEIYQPlZWV7N1biMVioU+fHqSmpnDo0JGw\ndZiuXp0LQK+e3Wq8rpRya37KDkssoXDOnxgZwpan/ojGuRRz5vzCkiVrSU1N4eab/uT1uAEDshhz\n0dkcPVrGM45mqmghiUIIHwoKDqC1Ji2tJfHx8fTOOhGAlSvD00/h6p/I6nbMe87mp2+mzQtLLMHa\nv/8gf/yxnqSkRM44o4+pZUXj7OynnnkHgL/fey0NGzbweeyTT9yBUoo33/qSbdt2hSM8v0iiEMIH\n54indu1SATjlFHuiWLFyfVjKdy4G6N6R7TR0aD+aNm3M2rWb2bx55zHvR4t5835Ha80ZZ/QxfNmO\n2qJtufGVq3L56aflpKQ05bZb/1zn8b16dePKK0ZTWWnlX08cMxYoYiRRCOGDs38ivV1rAHpn2T+w\nw1ejqDk01l1iYoKrgz2aRz85m53MGhbrrnoDo+ioUXz8sX0N1Lv/ehVNmzb265zHH7uN+Ph4Pvxo\nBjk54Z2z440kCiF82OW1RmH+yKcjR0rYsiWPxMQEunc/weMx0d5PobUOS0e2UzTNpfj999UsXWbf\nF/yuO6/w+7yuXTty041jsdlsPPrY6yZG6D9JFEL44KxROBPFiSd2IjExgS1b8kwfgrlmjb0jOzOz\nc41x9+5Gjz6DhIR4Fi36g/37D5oaTzC2bMlj27Z8WrRoRp8+PUwvL5pGPT39jL1D+o7bL6dFi2YB\nnfvIhPEkJycxZeqcqBjVJolCCB927bLXKJxNTwkJCfTs2RWouZmQGXz1Tzg1a9aEs4b2w2az8d13\nP5kaTzDmzLHXJoad3Z+4uDjTy3P2UeTl7aGqqsr08rxZuXID07/NJikpgXv+dnXA56enp3HnHeMA\nmPDI/4wOL2CSKITwobpG0dr1mqufwuxE4eifcJ9o58lY1x4V0ddPEa5hsU4NGiSTmppCZaWVPXsO\nhKVMT5zDWy84fzCtW7cM6hr3//MGmjRpxPezf2bhwmVGhhcwSRRC+JC/u2YfBbj1U6wwt5/CuQdF\nVh2J4qILzwJg9g+LKS0tMzWmQFRVVTFv/u8AjAhTogD3/bMj0/y0fv1Wvpwyh8TEBC7/87Cgr9Oq\nVQp/v/daAB6a8ErYJnl6IolCCB927XKMekp3r1E45lKsMi9RaK3dFgP03vQE0L59Gn37ZnL0aBk/\nOhbeiwbLl+dw8OBhMjLS6dy5fdjKjfTIp38/+x5aa264fgypqc1DutY9f7uali2b8/PPK5g1a5FB\nEQZOEoUQXpSUHKWo6AiJiQk1OiOdH9yrV+eatoXlzp0FFBUdITU1hbS0upsuonH0U7ibnZwiOZdi\ny5Y8Pvl0JnFxcdz/zxtCvl7Tpo158IEbAXj4kVcjtq6XJAohvNi9277eTrt2qTX2T0hJacoJJ7Sj\nrKycTZt2mFK2qyM7q7tfezc4Z2l/O2NBRDtx3Tn3nwjHsFh3kZyd/Z/nPqCqqoqrrzqfjAxjalG3\n33Y57dqlsmLFBqZMmWPINQMliUIIL1wjntyanZyqJ96Z0/zkmmhXR/+EU69e3ejUqR179xby22+r\nTYkpEEePlrJo0R8opRg2rH9Yy47Uek95eXv4YOI0lFI8+MBNhl23QYNkHp1wCwCPPPaaabVYXyRR\nCOGFa8RT22MThdkT7/wZGuvOvkigvVYRDaOfFi36g4qKSk499SRatgytnT5QrrkUeeFNFM+/MImK\nikouu3QkJ57YydBr33jjWLp06cDGjdv58MNvDb22PyRRCOGFpxFPTmYv5eHv0Fh3Y8c4FwmMfKJw\n7mYX7v4JiMyopz17DvD2O1MBePihvxh+/YSEBJ54/HYAHn/izbBvgSuJQggvfDU9nXKKfZaxGTWK\nsrJyNm7cjsViITOzs9/nnXFGH1q0aMbGjdtZv36r4XEFIlL9EwBpaS1JSIhn//6DHD1aGpYy//vS\nR5SWlnHRhWfVOUotWOPGjaJXr27s3FnAW29/aUoZ3hiSKJRSo5RSG5RSuUqpBzy8f5VSapVSarVS\narFSqrcR5Qphpuqmp2NrFJ06taNJk0YUFOw3fGJXTs4Wqqqq6N79hIBWW42Pj+f884YAkW1+2rev\nkBUrNpCcnMTpp58S9vItFotr5NPOnXtML6+wsIjXXv8cgIcfutm0ciwWC089Yd/c6Kmn36G4+Khp\nZR1TdqgXUErFAa8Bo4FM4AqlVGatw7YCQ7XWvYAngbdDLVcIszkXBPRUo7BYLPTubU6Htmv+RADN\nTk7V/RTZRoYUkLmOYbFDzuhDcnJSRGLoEMa5FK/+7zOKi49yzshB9O/fy9SyLrxwKAMG9GLfvoO8\n/MonppblzogaRX8gV2u9RWtdAUwGxrgfoLVerLV2rlj2KxC+2TdCBKn2goC1mbWUR/VmRYE3YZx7\n7mCSkhL59ddVEVvCwrWseAT6J5zCNenu8OFi1we2GX0TtSmleOapuwB47vlJFBYWmV4mGJMo0gH3\nXVPyHK95cxMwy4ByhTCN1trjOk/uTunt6KcweCmPUGoUjRs3ZPiwAWit+fbbBYbG5Q/7suKR68h2\nCtdy42+8+QUHDx5myJBTOfPMvqaW5TRs2ACGDx9AUdERnnt+YljKjA9LKQ5KqbOxJ4ozfBwzHhgP\nkJaWRnZ2dlBlFRcXB31uOMVCnMdjjIcPl1BWVk6jRsksXfq7x2Os1iMALP5lud9l+xPnsmVrACg5\nWhjU/6lHj3bMnAXvvf8lXbu2CPj8UO5lXt5eduzYTdOmjTh4cDfZ2eb0EdQVY2npYQB+//0P0352\ny8oqePb/3gPgwgsGeCzHrN+dP108mLlzf+O/L33Eaf0yaNGiqeFl1KC1DukLGATMdnv+IPCgh+Oy\ngM1Ad3+v3bdvXx2s+fPnB31uOMVCnMdjjKtXb9RYsnSPzDFejzl6tFRb4k/RcQl9dGlpmV/XrSvO\ngoL9GkuWbtp8sLbZbIGE7JKfv1djydJJDfrpI0dKAj4/lHv5+uufayxZ+s+X3xf0NfxRV4yzZi3S\nWLL08JF/MS2Gl17+WGPJ0qcNuMLr98rM350xY+/WWLL0nXc9E/Q1gKXaj89iI5qelgDdlFIZSqlE\nYBww3f0ApVRH4CvgGq11ePaQFCIEdfVPgH3G7IkndqKqqoq1azcbUq5zol2vXl39WrrDk7ZtUxk4\nMIvy8gp++GGxIXH5y7mb3cgRg8Jabm1mL+NRXl7Bf577AIAJD40P+nsViqeevIMrrziPu/96lell\nhZwotNZW4E5gNpADfKG1XquUulUpdavjsEeBlsDrSqkVSqmloZYrhJlqb1jkzSm9nUuOrzekXFdH\ntp8zsr2JxCKB9mXFlwAwYoT5+2P7Uj3qqcCU5bknTZpOfv4+srK6c8EFZxp+fX/07NmNTz7+N127\ndjS9LEPmUWitZ2qtu2utu2itn3a89qbW+k3H45u11ila61McX/2MKFcIs+TvrrtGAVQPkTVo5NPq\n1fbtT4PpyHbnHCY747uFYVsbaOnStRQVHaFr14506uRrPIv5mjRpREpKU8rLK9i3r9Dw60/+4nsA\n7rv3WiyW+j9vuf7/D4UIgj9NT+C+N4UxiSKUobHuevTIoFu3jhQWFvHzzyuMCK1OrmGxwyNbm3By\nr1UY6ciREhYt+gOLxcIFFww19NrRShKFEB5UNz2l+TzOuTjgypUbQ27isFqtrr4O577cwXJfJDBc\naz9FctkOT8yaSzFv3u9UVloZODCLlBSTRxtFCUkUQnjga0FAd23atKJ16xYUFR1h+/b8kMrcuHE7\nFRWVZGSk07Rp45CuBe79FPNN30azpOQoP/+8IiLLintj1lyK72f/DMCocwcbet1oJolCCA+cNYq6\nEgW4d2iHNvEu0KXF6zJoUG9SU1PYunUXa9bkGnJNb3766Q8qK63065cZNX9lu5YbNzBRaK2Z9b09\nUYwe5XU6WL0jiUKIWqqqqigosC9/0dbDgoC19e5tTD9F9dLioTU7OcXFxXGhow3d7EUCo2VYrDsz\nNjDasGEb27fnk5qawqmnnmTYdaOdJAohatm7txCbzUZqagqJiQl1Hm/UEFmjaxQQvkUCo60jG9zm\nUuw0ro9i1veLADj3nMHHxWgnp+PnfyqEn3ztQ+GJUUNkXdufGrifwYgRA2jQIJmlS9eya5c5y2ns\n2XOAVas20qBBMoMHh39ZcW/M2MDo+++d/ROnG3bNWCCJQohafG2B6smJJ3YiKSmRrVt3UVR0JKgy\nDx06zM6dBSQnJ9G1a4egruFJw4YNOGekvTlo+nRzFgl0Lit+5pBTSUpKNKWMYLRt24q4uDgKCvYb\nsiPc0aOlLFi4DKUU55wTPU1s4SCJoh7LydnC7Xc8TUHB/kiHElPy8/3vyAb7hkHO4azOlV8D5Zxo\n17NnV+Li4oK6hjfO0U/fTJtn6HWdXMNiI7harCfx8fGu72FeXui1qezspZSXV9CvXyapqYEvthjL\nJFHUY/fc+xxvvPkF9/79+UiHElN8bVjkjaufYmVw/RTVS3eENiPbkwsuOBOLxcL87CVB13i80VGy\nrLg3Rq75VD0s9vhqdgJJFKY5fLiY77//2fTx697s2LGbH+bYR6J8NnmW4buw1We+tkD1pnq3u+D6\nKVwd2Sbst5ya2oLBg3tTWWl1tbEbZePG7eTl7SE1NYVeJiS5UBk58ul4HBbrJInCBDabjTEX383o\n82/nww+/jUgMEydNQ2vtajOe8Mj/IhJHLPJ3+Q53rk2MgkzIrqGxIc7I9mbsGHNGPzmbnYYPGxCV\no4BccynyQksUubk7yM3dQUpKU/r372lEaDEl+r6z9cB7731NdrZ9gdz/vT457OXbbDY+mDjNHss7\nj9OoUQNmfLeQxYvDs+ZPrAum6Skry/7X9Jo1uQEvwmez2VjtmkNhzl/lzmGyM2ctorKy0rDrzpkT\nvc1OYNzIJ2ez0zkjBxnehxQLJFEYLD9/L/+4/78AJCTEs3TpWpYuXRvWGObN+51t2/I54YR2XHHF\naO7529UAPDTh1Yg1hcWSurZA9aRZsyZkZKRTXl7Bxo3bAypv69ZdlJSU0rZtqmmdpF27diQzszNF\nRUdYsGCZIde0Wq3Mz3YuKx6licKg9Z6O12GxTpIoDHbXX5+lqOgIF5x/JnfdeQVg31c3nN57/2sA\nbrh+DBaLhfv+fh0pKU1ZsGApPzo6HoVnZWXlHDhwiPj4eFJTUwI6t7ejfyHQpTyqJ9qZ28bvrFW8\n/sbnhtQqlixZy+HDxXTvfoLrL/doY8R6T2Vl5a59Ns49jtZ3cieJwkBfffUjX309l8aNG/L6aw9z\ny/hLAfhs8vccOnQ4LDEUFhbx9TfzUEpxw/VjAPtfuw/cfyMgtYq67HbsQ9G2bauA29yrl/IILFGs\nXmP8RDtPrr7qfBIS4vn6m3mcPfzmkCfgRdtqsZ50cKtRBPtz/9NPyyktLeOUU070a0mX+kgShUEO\nHTrMnX99FoBn/303HTq0oXv3TgwfPoDS0jI+/Cg8ndqffPod5eUVjBwxsMZfeXfeMY42bVqxdOla\nvv56blhiiUXBdGQ7Bbs4oHPuhdk1iszMLiyY/z7p6a35+ecVnNpvHPPm/Rb09aJx2Y7amjdvQuPG\nDSkpKeXgweD+WHMu23G8NjuBJArD3P/AS+zevY9Bg3pz261/dr1+2y32x2++NcX0v+S11rz3/jcA\n3HTjxTXea9iwAY9MGA/AhEdfo6qqytRYYpW/W6B6EuxSHkZtVuSPQYN688eyzxkxfCB79xYy8txb\n+fez72Gz2QK6TnHxUX75ZSUWi4Wzzz7NpGhDp5QKeS7F97Pt+44fj8NinQxJFEqpUUqpDUqpXKXU\nAx7eV0qpVxzvr1JKnWpEudFiwYKlvP3OVBIS4nn37cdqNFlcdNFQ2rZNJSdnCwsXGtOJ6M3y5Tms\nXLmBli2bM8YxHNLdzTf9iYyMdHJytvDxx9+ZGkus8ncLVE86dbLvI7FnzwG/Z8OXlBwlN3cn8fHx\n9OiREXCZwUhNbcH3s15nwsN/wWaz8dDDrzBm7N0B/cW9cOEyKiutnHbayTRvHh3LinsTylyK7dvz\nycnZQtOmjRk0KMvo0GJGyIlCKRUHvAaMBjKBK5RSmbUOGw10c3yNB94ItdxoUVZWzl9u+RcADz/0\nFzIzu9R4PyEhgZtvsv91b3antrMT++qrzve45k5iYgL/euw2AB771xuGrH9T3wQz4slJKeU28c6/\n5qe1azf/e/sgAAAgAElEQVSjtaZHj05+rVRrlLi4OJ584k5mTH+VlJSmzPhuIX1PG8cff+T4dX40\nz8auLZS5FM7RTiOGDyAhIXzfn2hjRI2iP5Crtd6ita4AJgNjah0zBvhQ2/0KNFdKRecwiQA9+dTb\nbNq0g8zMzq4O49r+cvMlWCwWvvp6Lnv2HDAljtLSMj79bBZwbLOTuyuvPI/MzM5s357Pu+99ZUos\nsSyUpidwX8rDv0RR3T9hfrOTJ+effybLl06mb99Mtm7dxaDTr+W7736ps5k0FjqynarnUgTe9HQ8\nL9vhzohEkQ7sdHue53gt0GNizsqVG/jPcxNRSvHu2497XTmzQ4c2XHjBUCorrbz/wTemxDJ16o8U\nFR3htNNO9jlpKy4ujqeeuBOwJ7mSkqOmxBOr/N0C1RvnEFl/l/Ko7p+I3PIXnTqls2jhRG4Zfynl\n5RU8/+JkbrzpUY4eLfV4fEHBftasyaVhw2QGDoz+5pjquRSB1SgqKipdHfajRh3fiSI+0gHUppQa\nj715irS0NLKzs4O6TnFxcdDn+qOqysYdd72I1Wrl4rFDKC8/6LO8wYNOZNr0+bz8ysf0Py2DuDiL\noXG+8OIHAJxx+sl1Xq95cws9TuzI+g07uOfep7nyipE+jzf7XhrBqBg35+4AIC9vG9nZZQGfb60q\nAWDx4uUe46kd58Kf7OPz0WURv8fjLh9CSkoSL/73CyZOms5Pi5bxr8duID29ZtKcM8cec6+eGfzy\ny+JIhBrQ9/tAoX0Y8Oo1GwK6xytWbKK4+CidOrVh8+b1bN4c+IKPsfC74xetdUhfwCBgttvzB4EH\nax3zFnCF2/MNQNu6rt23b18drPnz5wd9rj9eeHGSxpKlO5xwjj58uLjO46uqqnTnrudpLFl6xowF\nrteNiDM3d4fGkqUbNOqvDx067Nc5c+b8orFk6eYtTtcHDxb5PNbse2kEI2K02Wy6UZMBGkuW3/ex\nttLSMh2X0Edb4k/RR4+WHvO+e5w2m023aDVEY8nSeXkFwYZtuHff/Vh37X6BxpKlmzYfrL/+em6N\n96+7foLGkqVfeHFShCIM7Pu9efNOjSVLt+84MqAy/nn/ixpLlv77fc8HGF21aP/dAZZqPz7njWh6\nWgJ0U0plKKUSgXHA9FrHTAeudYx+GggUaa2N258wzLZsyWPCI68B8MZrD9OkSaM6z7FYLK4JeEZ3\najubsy67dCTNmjXx65zhwwdw9tmncejQEZ5/4UND44lVR46UUFJSSsOGyTRt2jioayQnJ9GjRyds\nNhtr1uT6PDY/fy+FhUW0aNEsqM5zs3Tpks7S3z/l4rHDOHy4mIsvuYf7H/gvVqsVrbWrfyKa9sf2\nJT29NUop8vP3BTQjXYbFVgs5UWitrcCdwGwgB/hCa71WKXWrUupWx2EzgS1ALvAOcHuo5UaK1ppb\nb3uS0tIyrhg3mvPPP9Pvc2+4fgyJiQnMnLWI7dvzDYnHarUycZJ9AUBfndi1KaV45qm/AvDSyx+b\n1skeS5wd2e3a2T9YgtU7yzFDu45+CmdHdq9e3UIqzwzNmjVh6pQXef65e4mLi+M/z01kxDm3kJ29\nhF279tK6dQvXZk3RLikpkTZtWmGz2Vyj2uqSn7+XVas20qhRA844o4/JEUY/Q+ZRaK1naq27a627\naK2fdrz2ptb6TcdjrbW+w/F+L631UiPKjYSPPprBnB9/pUWLZrz0338EdG5qagsuvWQkWmvefmeq\nIfHMnr2Y/Px9dOvWkSFDApueMnBgFhddeBYlJaU88+93DYknljk/RAJZNdaTU07xbxMjMzcrMoJS\nir/fex3z575L27apLFiwlBHn3ALYRztFW3LzpXrSnX8d2s5hscPO7h9V27tGiszMDsDevQe45+/P\nAfDfF/5B69YtA77GbbdeBsC7731FRUXoC7M5507cdOPFQf3iPvnEHSilePOtLw2r5cQq1xaoIa7n\n4xr5VMcM7UgPjfXXkCGnsnzpZIYO7eeawR0L8yfcOSfd7fRzcUBns9PxPtrJSRJFAP52z3MUFhYx\ncsRArrnmgqCucfrpfTj55C7s3VvIN9+Etofxnj0H+HbGQuLi4rju2ouCukZWVneuGDeaiopKnnjy\nrZDiiXW7Atwr2xvX4oArN/pcGiMahsb6q02bVvz4w1s8+sgtDB8+gLFjj535H81cNYqddXeNWq1W\n5vxo3x3yeJ8/4SSJwk/ffbeQzybPomHDZN5685Ggq91KKddaUG+8FVqn9ocffYvVauX884bQpk2r\noK/zr8dvIz4+nomTprNhw7aQYjKa1pqb//I4110/wfS1soxqekpLa0mbNq04cqSEbds819LKyytY\nv34bSilOPrmLx2OiTXx8PP96/HZ+/OHtqF+2o7ZAmp5++201hw4doXv3E+jcub3ZocUESRR+OHKk\nhNvueBqwN9VkZIT2w3PN1RfQqFEDsrOXsn17cOvka61rNDuFomvXjtx041hsNhuPPPpaSNcy2owZ\nC3jv/a/58KNv/W42CFYwe2V7Uz3xzvMM7fXrt2K1WunatQONGjUMuTzhW4f2/i8MOOs436TIE0kU\nfnh4wqvs3FlAv34n89e7rgz5ek2bNubKK84D4NsZwU1YWrx4BRs2bKNNm1acd17ow/cemTCepKRE\nvpzyA8uX+7fej9lsNhsT3BJXXcNNQxXMFqjeVHdoe04Uzq1Po71/or4IZAMj57IdMiy2miSKOvzy\ny0r+99pk4uLiePftx4iPN2Yyu7NTe/YPv3tdKsEXZ23iumsvNCSm9PQ07rxjHAATHvlfyNczwhdf\nzHbt/gbVG/yYJZQFAWura4jsKpP3yBY1+dv0tHfvAZYtW0dychJDh/YNR2gxQRKFDxUVldw8/nG0\n1vzzH9e7OimN0KfPSQwY0Ivi4lImT/4+oHOPHCnhiy9/AODGG8YaFtMD999IkyaNmPX9In76ablh\n1w2G1Wrl0cdfB+x7KIC5NQqbzVZjd7tQ1VWjCNf2p8KuVasUkpOTKCo6wuHDxV6Pm+0Y7XTW0H40\naJAcrvCiniQKH579v/dYt24L3bp1dG36Y6Rbx9trFW+89WVA533++WxKSkoZMuRUunfvZFg8rVql\n8Pd7rwXgoQmvRHTL1EmTprNp0w66devI//37bwCsNjFR7N9/kMpKKy1aNDPkA6Jbt44kJyexfXu+\nx21wnTWKcGxWJGpuYOSrr0uGxXomicKLdes289TT7wDwzluPmfLXxeWXn0uTJg1ZunQtS5eu9fu8\n9z5wdGLfEFontif3/O1qWrZszqJFf7gmHYVbWVk5/3IM1f3XY7e7/jrPydmC1Wo1pcxQtkD1JD4+\nnl697DOXazc/7dtXyO7d+2jUqAEZGTG/iHLMqGsDo6qqKmb/4Fy2QxKFO0kUbg4fLub331czceI0\nrr72ISorrfzl5ksYOrSfKeU1aJDMuef0B+BNP2sV69Zt5tdfV9GkSSMuvXSE4TE1bdqYBx+w76vx\n8COvBrxFphHeensKO3cW0KtXN0cybUSnTu2oqKhk06YdppTpWr7DgBFPTq5+ilU1m59Wu/VPuO+G\nKMxV15aoy5at48CBQ2RkpNOt2wnhDC3qRd0y4+Gwf/9BcnK2sm7dZnLWV/+bl7enxnFt26byn//7\nm6mxXHjhYKZMzebTz2bx/HP31jk+3dmJfcW4UaYNq7z9tsv570sf88cf65kyZQ6tWyeZUo4nxcVH\nXcuJPPXEna4P0p4nd2XbtnzWrMnlpJM6G16uUXMo3Ln6KVbUTBSuZifpnwirukY+OYfFjh51Rkwt\nTxIO9TZRaK3ZtWuPPSHkbHb8u4WcnC3s23fQ4zlJSYmceGInMk/qzEknZXDtNReaPrGoY4c0hg3r\nz7x5v/PRxzO4607vw28rKir58KMZQOhzJ3xp0CCZRyfcwi23Pckjj73G6/+727Syanvl1U/Zu7eQ\nAQN6ceGFQ12v9+zZlRnfLWT1mk1cdtk5hpdbvVe2cYnC21Iero5s6Z8Iqw7t0wDvNYrq3ewGhy2m\nWFEvE8XV1zzIN9PmUVLieeOZxo0bctJJGWSe1MXxb2dOOqkzGRnpxMXFhTlauO2WPzNv3u+88eaX\n3HnHFV7/mvn22wXs33+Qnj27ctppPU2N6YYbxvCf5yeyceN2Zv+whOHDh5taHsDBg4d57vmJADz9\n5F017kOvnva/vs0a+VS9cqxxTU/ORLBmTS6VlZWuPZednfIyhyK8fNUoDhw4xO+/ryExMYGzz+4f\n7tCiXr1MFOXllZSUlNGiRTMyMztzUo/Ojn8zyMzsQvv2aVFVtRwz5izatGlFTs4WFi5c5rVPJNQF\nAAORkJDAE4/fzlXXPMgHH8zkH/eNJzW1hallPv/CJA4dOsKwYf0ZPnxAjfecS1qvWWtOonAuCBjs\nXtmeNG3amM6d27NlSx4bNmyjZ89uVFVV71MRK8t01xe++ijmzPkFm83GkCGn0bixzJSvrV72pD33\nn3v4aspT7N+7gJ8WTOTttx7lb3dfzbnnnk6HDm2iKkmA/UP55pvsTUneOrXz8vYw+4fFJCTEc/VV\n54clrnHjRnH66aew/0AR466837QRR2Bf4PDlVz4B7LWJ2nr0yCA+Pp7c3J1BTVCsS3XTk3E1CoBT\neteceLcrfx9lZeV06NCGlJTYWi8p1nVw7J2dl7eXqqqqGu+5hsXKsh0e1ctE0alTOikpTaIuIfjy\nl5svwWKxMPWrHz1uIjRx0jRsNhtjxwyjVauUsMRksVj4YvJzpKQ0Yd6833no4VdNK+vfz75HSUkp\nF14wlIEDs455PzExge7dO6K1Jidnq+Hlm9H0BNC7t715yTnxbssW+yKB0pEdfg0aJJOamoLVaq3x\nO2az2dyW7ZBE4Um9TBSxqGPHtpx/3hAqK62urU2dbDab67WbbjRuJrY/2rVrzeOP3kB8fDzPPT+R\nLx0zwo20Y8du1/awTz5xh9fjnP0UzuGlRqmsrGTv3kIsFgtpaYHvMeLLKb17ANWLA7oShXRkR4Sr\nn8JtLsXKlRvYs+cA7dunkZkZGyv5hpskiijiXH787Xem1KgaZ2cvYevWXXTo0IYREdgwJiurCy88\nfy8AN9z0KGsN7id48qm3qaioZNzlo3wuk2JWP0VBgf2vy7S0loat5eXkXqPQWrNZahQR1bHDsf0U\nMiy2biElCqVUC6XUHKXUJse/x7SJKKU6KKXmK6XWKaXWKqXCN9Yyxpx77mA6dWrHtm35rjVnAN57\n316buOH6MREZlQVw151XctWV51FSUsrFl9xDUdERQ667adN2Ppg4jbi4OP71+G0+jzWrRuFsdjJy\nDoVTx45tad68Cfv2HaSgYL/UKCLM2U/hPvJJhsXWLdQaxQPAXK11N2Cu43ltVuDvWutMYCBwh1Iq\nM8Ry6yWLxcIt4y8FcDXFHDx4mKlf/YhSihuuHxOx2JRSvP3Wo/TufSKbNu3g2usmGDJr+7HH36Cq\nqorrr7uoznWrzKpRGLUFqidKKVctaeHCZRQUFDr6W2TmbyTUHvl06NBhFi9eSXx8/DEj7US1UBPF\nGGCS4/Ek4JgGdK31bq31csfjI0AOIAvceHHjDWNJSIjnu5k/sX17Pp9+NpPy8gpGDB9Ap06RvW0N\nGzbgqykv0rx5E6Z/m+2aQR2sVas2Mvnz70lMTODRR26p8/iMjHQaNkwmP38fhYVFIZXtzqwRT07O\niXeffDoTgMzMzoY3cQn/1F7vae7c36mqqmLw4N40a9YkkqFFtVATRZrW2tnYVwCk+TpYKdUJ6AP8\nFmK59Vbr1i259JKRaK15592vDNvFziidO7fn04+fRSnFo4+9zqxZi4K+1iOPvobWmltvuczVyeiL\nxWJxbRtq5MQ7M5ueoHqIrLMtXCbaRU7tvbOrm51ktJMvdf5Zo5T6EWjj4a2H3Z9orbVSyuu61Eqp\nxsBU4G9a62PXXa4+bjwwHiAtLY3s7Oy6QvSouLg46HPDyVOcAwZ05bPJ8MKLkygrq6Bpk4akpMRH\n7P9TO8YGDeCG60fz/gcz+fO4+3jrjfto1y6wPRzW5Wxj+rfZJCcnMvTMk/z+v7Vs2RiAqV/NxGar\n7icJ5fu9fPlqAA4fLjTlHldW2uN0zkNp1NAS1T+bsfC7E2yMBw7Ya6KbN+9k/vz5fDNtLgCprZJM\n+T/Hwr30i9Y66C9gA9DW8bgtsMHLcQnAbODeQK7ft29fHaz58+cHfW44eYrTZrPpk3tdrLFkaSxZ\n+q93Pxv+wNx4irGqqkpfNOavGkuW7t3nMl1ScjSgaw4f+ReNJUs/+NDLAZ334n8/1Fiy9K23PVln\njIHGMmvWoqCv4UtpaZmOTzzV9f384YfFppRjlFj43Qk2xqqqKp2QZP9e/PbbKo0lS7dpN0zbbDZj\nA3SI9nsJLNV+fBaH2vQ0HbjO8fg6YFrtA5R9vNl7QI7W+sUQyzsuKKW49ZbLXM+jpdnJncVi4cNJ\nT9GtW0dWrtzA+Fue8Hujo3nzfmPu3N9o1qwJ/7jv+oDK7Xmy8R3aZqwc6y45OYkePTq5nsuIp8ix\nWCyukU9vvzMVsI92kmGxvoWaKJ4FRiqlNgEjHM9RSrVTSs10HHM6cA0wTCm1wvF1Xojl1nvXXH0B\nXbt25MILhkbtB0uzZk34eup/adSoAZ98OpP/vfZZnedorXnYsSf3P+67LuBlLJx7TK9enWvYDnxG\nb1rkibOfIqV5Y8Mn9YnAOPspPps8C5D+CX+ElCi01ge01sO11t201iO01oWO1/O11uc5Hi/SWiut\ndZbW+hTH10zfVxbNmjVh04ZvmT7tlUiH4tPJJ3flg/eeAODev79Q517b3323kF9/XUVqagp3//Wq\ngMtLS2tJy5bNKSo64uqEDkVJyVGKio6QlJRIixbNQr6eN84hsp07tzOtDOGfDu3tieLo0TIsFgsj\nRw6KcETRT2Zmi5Bddtk5/OO+67FarVx2+X2ueQm12Ww2V23ioQdvDmqVTqWUa4tRIybeudcmzGx+\nGHf5KAYOzGLMmDNMK0P4x32E3YABvUz9A6G+kEQhDPHM03cxbFh/9uw5wKV/vo+Kispjjvnyyx9Y\ntWoj7dun1eiDCZSR/RThaHYCaN8+jV9+/oghZ/Q2tRxRN+cyHiCzsf0liUIYIj4+nsmf/h8dOrTh\nl19Wcs+9z9V432q18ujjrwPw6IRbSE4OfntVI5fyqN4r25yObBF93GsUo0dJDc8fkiiEYVJTW/DV\nlBdJSkrk9Tc+Z+LE6kFwH374LRs3bqdLlw5cf/1FIZVj5FIezlnZZo14EtGnS5f2AKSmptC3r6wm\n5A9JFMJQ/fqdzOv/ewiAW29/iuXLcygvr+BfT74FwBOP3+7aEjRYzkSxbt2WkDdTcq3zZHLTk4ge\n3bqdwDtvPcaXnz+PxSIfgf6QuyQMd+ONF3PL+EspL6/g4kvu4Zl/v8uOHbvp2bMr48aNCvn6TZs2\npmPHtpSXV7B5c15I13It32HgFqgi+t1885+8bjksjiWJQpji5ZfuZ8CAXuzYsZsnHLWJp56407C/\n4Hr1NGbkk9kLAgpRH0iiEKZISkpkyhcv0Lp1CwD69+/JRRedZdj1jeqnMGsLVCHqE0kUwjTt26cx\n7euXGTF8IG+8NsHQeQquIbIhrCKrtXYbHitNT0J4I4viC1MNHJjFnB/eMvy6rqU81gTf9HTw4GHK\nyyto2rRxUJP/hDheSI1CxKQePTKIi4sjN3cnpaVlQV1Dmp2E8I8kChGTkpIS6d79BGw2Gzk5W4O6\nhnNorMyhEMI3SRQiZoXaT+HqnzBhr2wh6hNJFCJmuRYHDLKfYpdMthPCL5IoRMwyqkYhTU9C+CaJ\nQsSsUEc+VTc9SaIQwhdJFCJmZWSk06BBMrt27eXIkaMBny9NT0L4RxKFiFlxcXGcfHIXALZu3R3w\n+TLqSQj/hJQolFItlFJzlFKbHP+m+Dg2Tin1h1JqRihlCuGupzNRbAssUVitVgoKDgDQpk0rw+MS\noj4JtUbxADBXa90NmOt47s3dQE6I5QlRg3PNp0BrFHv3FmKz2WjdugWJiaEtey5EfRdqohgDTHI8\nngSM9XSQUqo9cD7wbojlCVGDc7e7QBNFuLZAFaI+CDVRpGmtnb+hBUCal+NeAv4J2EIsT4gaXDWK\nbbvRWvt9nmyBKoT/6lwUUCn1I9DGw1sPuz/RWmul1DG/qUqpC4C9WutlSqmz/ChvPDAeIC0tjezs\n7LpO8ai4uDjoc8MpFuKM5hi11jRt0pDDR44yZco0UlOb+3Ve9oKfAVAWa1j/b9F8L50kRuPESpx1\n0loH/QVsANo6HrcFNng45t9AHrANe63jKPCxP9fv27evDtb8+fODPjecYiHOaI/xzLNu0Fiy9Pff\nL/L7nAmPvKqxZOnHHn/dxMiOFe33UmuJ0UjRHiewVPvxWRxq09N04DrH4+uAaR4S0YNa6/Za607A\nOGCe1vrqEMsVwsW1210AM7Srm56kj0KIuoSaKJ4FRiqlNgEjHM9RSrVTSs0MNTgh/BHMUh6yfIcQ\n/gtp4yKt9QFguIfX84HzPLyeDWSHUqYQtQWzlIfslS2E/2Rmtoh5ztnZ69Ztoaqqyq9zqjctkhqF\nEHWRRCFiXvPmTWndujllZeVs3ryzzuPLysopLCwiPj6e1FSviwkIIRwkUYh6IaNTW8C/fgpn/0Tb\ntq2wWORXQIi6yG+JqBcyMtoBsGatP4lCVo0VIhCSKES9kJFhr1GsXu1/jSJd+ieE8IskClEvuJqe\n/KhRyD4UQgRGEoWoF044IQ2LxcKmTTsoKyv3eazMoRAiMJIoRL2QmJhAt24dqaqqYv36rT6PlS1Q\nhQiMJApRbziXHF+92vfEO2l6EiIwkihEveFccryufgrZAlWIwEiiEPWGa3FAHyOftNayaZEQAZJE\nIeoNf2oUhw8XU1JSSsOGyTRt2jhcoQkR0yRRiHqjS5cOJCcnsXNnAYcOHfZ4jPuIJ6VUOMMTImZJ\nohD1RlxcHJmZnQFYu3azx2NkMUAhAieJQtQrzpFP3tZ8qh4aK/0TQvhLEoWoV3q6drvzPEQ2f7eM\neBIiUJIoRL3i2u2uzqYnqVEI4S9JFKJe6dXLOUR2E/a942uSBQGFCFxIiUIp1UIpNUcptcnxr8dd\nYJRSzZVSU5RS65VSOUqpQaGUK4Q37dq1pnnzJhQWFlFQsP+Y92ULVCECF2qN4gFgrta6GzDX8dyT\nl4HvtdY9gN5ATojlCuGRUqp6D20PS3nIqCchAhdqohgDTHI8ngSMrX2AUqoZcCbwHoDWukJrfSjE\ncoXwqrqfoubIJ5vNxu7d9lqG1CiE8F+oiSJNa73b8bgASPNwTAawD/hAKfWHUupdpVSjEMsVwitv\niwPu338Qq9VKixbNSE5OikRoQsSk+LoOUEr9CLTx8NbD7k+01lopdWzvob2MU4G7tNa/KaVext5E\n9YiX8sYD4wHS0tLIzs6uK0SPiouLgz43nGIhzliLsbLyCAC//PpHjbg3bcoDoFmzhhH7/8TavYxW\nsRAjxE6cddJaB/0FbADaOh63BTZ4OKYNsM3t+RDgO3+u37dvXx2s+fPnB31uOMVCnLEWY2FhkcaS\npRs06q+tVqvr9RkzFmgsWfrcUbdGIEK7WLuX0SoWYtQ6+uMElmo/PotDbXqaDlzneHwdMM1DIioA\ndiqlTnS8NBxYF2K5QniVktKU9PTWlJaWsXXrLtfrModCiOCEmiieBUYqpTYBIxzPUUq1U0rNdDvu\nLuATpdQq4BTgmRDLFcInT0t5VA+NlRFPQgQipEShtT6gtR6ute6mtR6htS50vJ6vtT7P7bgVWut+\nWussrfVYrfXBUAMXwhdPS3nIZDshgiMzs0W95Boi61ajkKYnIYIjiULUS65JdzWaniRRCBEMSRSi\nXjrppAwsFgsbN26nvLwCqLlpkRDCf5IoRL3UoEEyXbt2oKqqivXrt1JRUcnevYVYLBZat24R6fCE\niCmSKES95d5P4VwgsE2blsTH1znPVAjhRhKFqLeq+yk2Ve9sJyOehAiYJApRb7nXKFwjnmQLVCEC\nJolC1FvuI59kC1QhgieJQtRbXbq0JykpkR07dpOTsxWQobFCBEMShai34uPjyczsDMCcH38FpOlJ\niGBIohD1mrOfIjd3ByBNT0IEQxKFqNec/RROMupJiMBJohD1mrNG4SR9FEIEThKFqNecq8gCJCUl\n0qJFswhGI0RskkQh6rX27dNo1qwJYK9NKKUiHJEQsUcShajXlFL07NkFkGYnIYIliULUe87d7mTD\nIiGCI4lC1HuDB/UGIDOzS4QjESI2yTKaot678srzyMhIp1+/kyMdihAxKaQahVKqhVJqjlJqk+Pf\nFC/H3aOUWquUWqOU+kwplRxKuUIEIi4ujjPOOJXk5KRIhyJETAq16ekBYK7Wuhsw1/G8BqVUOvBX\noJ/WuicQB4wLsVwhhBBhEmqiGANMcjyeBIz1clw80EApFQ80BPJDLFcIIUSYKK118CcrdUhr3dzx\nWAEHnc9rHXc38DRQCvygtb7KxzXHA+MB0tLS+k6ePDmo2IqLi2ncuHFQ54ZTLMQpMRonFuKUGI0T\n7XGeffbZy7TW/eo8UGvt8wv4EVjj4WsMcKjWsQc9nJ8CzANSgQTgG+DqusrVWtO3b18drPnz5wd9\nbjjFQpwSo3FiIU6J0TjRHiewVPvxWVznqCet9Qhv7yml9iil2mqtdyul2gJ7PRw2Atiqtd7nOOcr\nYDDwcZ1ZTAghRMSF2kcxHbjO8fg6YJqHY3YAA5VSDR3NU8OBnBDLFUIIESahJopngZFKqU3Yaw7P\nAiil2imlZgJorX8DpgDLgdWOMt8OsVwhhBBhEtKEO631Aew1hNqv5wPnuT1/DHgslLKEEEJERkij\nnsymlNoHbA/y9FbAfgPDMUssxCkxGicW4pQYjRPtcZ6gta5ztcyoThShUEot1f4M+4qwWIhTYjRO\nLIBbHJoAAASHSURBVMQpMRonVuKsiywKKIQQwidJFEIIIXyqz4kiVkZWxUKcEqNxYiFOidE4sRKn\nT/W2j0IIIYQx6nONQgghhAFiPlEopUYppTYopXKVUp6WOVdKqVcc769SSp0a5vg6KKXmK6XWOfbk\nuNvDMWcppYqUUiscX4+GM0a3OLYppVY7Yljq4f1I38sT3e7RCqXUYaXU32odE5F7qZR6Xym1Vym1\nxu01f/dr8fkzbHKMzyml1ju+n18rpY5Z1NNxnM+fDZNjfFwptcvte3qel3PDch99xPm5W4zblFIr\nvJwblntpKH8WhIrWL+x7W2wGOgOJwEogs9Yx5wGzAAUMBH4Lc4xtgVMdj5sAGz3EeBYwIwru5zag\nlY/3I3ovPXzvC7CPA4/4vQTOBE4F1ri99h/gAcfjB4D/8/L/8PkzbHKM5wDxjsf/5ylGf342TI7x\nceA+P34ewnIfvcVZ6/0XgEcjeS+N/Ir1GkV/IFdrvUVrXQFMxr6qrbsxwIfa7leguWMBw7DQWu/W\nWi93PD6CfZ2r9HCVb7CI3stahgObtdbBTsg0lNZ6IVBY62V/9mvx52fYtBi11j9ora2Op78C7c0o\n219e7qM/wnYfwXecjjXt/gx8Zlb54RbriSId2On2PI9jP4T9OSYslFKdgD7Abx7eHuyo/s9SSkVq\nc2cN/KiUWubYF6S2qLmX2HdJ9PaLGA33EiBNa73b8bgASPNwTDTd0xux1xg9qetnw2x3Ob6n73tp\nwoum+zgE2KO13uTl/Ujfy4DFeqKIGUqpxsBU4G9a68O13l4OdNRaZwGvYt+zIxLO0FqfAowG7lBK\nnRmhOHxSSiUCFwFfeng7Wu5lDdre5hC1QwyVUg8DVuATL4dE8mfjDexNSqcAu7E360SzK/Bdm4iJ\n3zN3sZ4odgEd3J63d7wW6DGmUkolYE8Sn2itv6r9vtb6sNa62PF4JpCglGoVzhgdZe9y/LsX+Bp7\ndd5dxO+lw2hgudZ6T+03ouVeOuxxNs0p7/u1RPyeKqWuBy4ArnIktGP48bNhGq31Hq11ldbaBrzj\npeyI30cAZd/u+U/A596OieS9DFasJ4olQDelVIbjr8xx2PfIcDcduNYxYmcgUOTWHGA6R3vle0CO\n1vpFL8e0cRyHUqo/9u/LgXDF6Ci3kVKqifMx9k7ONbUOi+i9dOP1L7ZouJdu/NmvxZ+fYdMopUYB\n/wQu0lof9XKMPz8bZsbo3g92sZeyI3of3YwA1mut8zy9Gel7GbRI96aH+oV9JM5G7CMeHna8ditw\nq+OxAl5zvL8a6Bfm+M7A3uSwCljh+DqvVox3Amuxj9T4FRgcgfvY2VH+SkcsUXcvHTE0wv7B38zt\ntYjfS+yJazdQib19/CagJTAX2IR9S+EWjmPbATN9/QyHMcZc7G37zp/NN2vH6O1nI4wxfuT4eVuF\n/cO/bSTvo7c4Ha9PdP4suh0bkXtp5JfMzBZCCOFTrDc9CSGEMJkkCiGEED5JohBCCOGTJAohhBA+\nSaIQQgjhkyQKIYQQPkmiEEII4ZMkCiGEED79P9BLryuDC+PGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc218e95a90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEJCAYAAABVFBp5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFEZJREFUeJzt3H+MZeV93/H3p7umqahTm+KM10ALVlAqWtUET4iVUmuo\nwcIQeaGNWqzGJnWqFWpw40hRshVS5Mr/gCs3USpkuk5RSJuEqE2xV7DBAcoVqhS7LC5ewIawQVRm\nvbA1Vu2OUwVjf/vHnG1vhjszd55z5t476/dLuprz43nO+d7nPLMfzrlzSVUhSVKLvzDvAiRJu5ch\nIklqZohIkpoZIpKkZoaIJKmZISJJamaISJKaGSKSpGaGiCSp2d55F9Di3HPPrQsvvLC5/7e//W3O\nPvvs4QraAdY4nN1QpzUOZzfUuRtqfPzxx79eVW/ZsmFV7brXO9/5zurjkUce6dV/FqxxOLuhTmsc\nzm6oczfUCBytKf499nGWJKmZISJJamaISJKaGSKSpGaGiCSpmSEiSWpmiEiSmhkikqRmu/Ib631c\nePB+AF5Y6dn/tuv619DjGGeCvuOwCNdi0BpWmg+xWO9jF8/rRRiHRahhO7wTkSQ1M0QkSc0MEUlS\nM0NEktTMEJEkNTNEJEnNDBFJUrNBQiTJNUmeTXI8ycFN2v1YkteS/NTYtheSPJnkiSRHh6hHkjQb\nvb9smGQPcAdwNfAi8FiSw1X15Qntbgf+cMJhrqyqr/etRZI0W0PciVwOHK+q56vqVeAeYP+Edh8B\nfh84NcA5JUkLYIj/7cl5wFfH1l8Efny8QZLzgBuAK4EfW9e/gIeSfBf4t1V1aNJJkhwADgAsLS0x\nGo16FT3v/lsdY3V1dZBz7KShatzpazFNnYs+H2Z1jCHm5Lzn7RDzchF+vxdhPkxjVv/vrF8Dfrmq\nvpdk/b4rqupEkh8CHkzyTFU9ur5RFy6HAJaXl2tlZaWtkgfW/p8yc+s/5TFGo1G/c8xA7xpndC02\nrXOXzIcdP8YQc3KI9zGAXvNyEX6/F2E+bMMQIXICuGBs/fxu27hl4J4uQM4Frk3yWlV9pqpOAFTV\nqST3svZ47HUhIklaPEN8JvIYcHGSi5KcBdwIHB5vUFUXVdWFVXUh8J+Af1ZVn0lydpI3AiQ5G3gv\n8NQANUmSZqD3nUhVvZbkFuBzwB7grqp6OsnN3f47N+m+BNzb3aHsBX6nqh7oW5MkaTYG+Uykqo4A\nR9ZtmxgeVfUzY8vPA+8YogZJ0uz5jXVJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0Qk\nSc0MEUlSM0NEktTMEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0Qk\nSc0MEUlSM0NEktRskBBJck2SZ5McT3Jwwv79SY4leSLJ0SRXTNtXkrS4eodIkj3AHcD7gEuADyS5\nZF2zh4F3VNWlwIeB39hGX0nSghriTuRy4HhVPV9VrwL3APvHG1TValVVt3o2UNP2lSQtriFC5Dzg\nq2PrL3bb/pwkNyR5BriftbuRqftKkhbT3lmdqKruBe5N8m7g48BV2+mf5ABwAGBpaYnRaNSrnnn3\n3+oYq6urg5xjJw1V405fi2nqXPT5MKtjDDEn5z1vh5iXi/D7vQjzYRpDhMgJ4IKx9fO7bRNV1aNJ\n3p7k3O30rapDwCGA5eXlWllZaav2gfsBmFv/KY8xGo36nWMGetc4o2uxaZ27ZD7s+DGGmJNDvI8B\n9JqXi/D7vQjzYRuGeJz1GHBxkouSnAXcCBweb5Dkh5OkW74M+IvAK9P0lSQtrt53IlX1WpJbgM8B\ne4C7qurpJDd3++8E/gHwoSTfAf4P8I+6D9on9u1bkyRpNgb5TKSqjgBH1m27c2z5duD2aftKknYH\nv7EuSWpmiEiSmhkikqRmhogkqZkhIklqZohIkpoZIpKkZoaIJKmZISJJamaISJKaGSKSpGaGiCSp\nmSEiSWpmiEiSmhkikqRmhogkqZkhIklqZohIkpoZIpKkZoaIJKmZISJJamaISJKaDRIiSa5J8myS\n40kOTtj/N5L8UZI/S/KL6/a9kOTJJE8kOTpEPZKk2djb9wBJ9gB3AFcDLwKPJTlcVV8ea/YN4J8D\n129wmCur6ut9a5EkzdYQdyKXA8er6vmqehW4B9g/3qCqTlXVY8B3BjifJGlBDBEi5wFfHVt/sds2\nrQIeSvJ4kgMD1CNJmpHej7MGcEVVnUjyQ8CDSZ6pqkfXN+oC5gDA0tISo9Go10nn3X+rY6yurg5y\njp00VI07fS2mqXPR58OsjjHEnJz3vB1iXi7C7/cizIdpDBEiJ4ALxtbP77ZNpapOdD9PJbmXtcdj\nrwuRqjoEHAJYXl6ulZWVtmofuB+AufWf8hij0ajfOWagd40zuhab1rlL5sOOH2OIOTnE+xhAr3m5\nCL/fizAftmGIx1mPARcnuSjJWcCNwOFpOiY5O8kbTy8D7wWeGqAmSdIM9L4TqarXktwCfA7YA9xV\nVU8nubnbf2eStwJHgR8Evpfko8AlwLnAvUlO1/I7VfVA35okSbMxyGciVXUEOLJu251jyy+x9phr\nvW8B7xiiBknS7PmNdUlSM0NEktTMEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS\n1MwQkSQ1M0QkSc0MEUlSM0NEktTMEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS\n1MwQkSQ1GyREklyT5Nkkx5McnLA/SX69238syWXT9pUkLa7eIZJkD3AH8D7gEuADSS5Z1+x9wMXd\n6wDwqW30lSQtqCHuRC4HjlfV81X1KnAPsH9dm/3Ab9WazwNvSrJvyr6SpAWVqup3gOSngGuq6p92\n6x8Efryqbhlrcx9wW1X91279YeCXgQu36jvJ8vJyHT16tLXgtn6StNv0+Pc9yeNVtbxVu73NZ5ix\nJAdYexTG0tISo9Go6Tgrw5UkSQut9d/J7RgiRE4AF4ytn99tm6bNG6boC0BVHQIOwdqdyMrKSlu1\nVYxGI5r7z8hmNV548H4AXrjtuubj9z3GEDXMym6/3otip2tchHk9K7O63jt/hmE+E3kMuDjJRUnO\nAm4EDq9rcxj4UPdXWu8CvllVJ6fsK0laUL3vRKrqtSS3AJ8D9gB3VdXTSW7u9t8JHAGuBY4Dfwr8\nk8369q1JkjQbg3wmUlVHWAuK8W13ji0X8HPT9pUk7Q5+Y12S1MwQkSQ1M0QkSc0MEUlSM0NEktTM\nEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSM0NEktTM\nEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDXrFSJJzknyYJLnup9v3qDdXUlOJXlq3faPJTmR5InudW2f\neiRJs9X3TuQg8HBVXQw83K1P8pvANRvs+9WqurR7HelZjyRphvqGyH7g7m75buD6SY2q6lHgGz3P\nJUlaMH1DZKmqTnbLLwFLDcf4SJJj3SOviY/DJEmLae9WDZI8BLx1wq5bx1eqqpLUNs//KeDjQHU/\nPwl8eIM6DgAHAJaWlhiNRts81f+3urraq/8sTFPjEO+h7zEWfRzhzLne8zarGhdhXu+03XC9p7Vl\niFTVVRvtS/Jykn1VdTLJPuDUdk5eVS+PHevTwH2btD0EHAJYXl6ulZWV7ZzqzxmNRvTpPwub1vjA\n/QD93kPfYwxRw4zs+uu9IHa8xkWY1zOyG673tPo+zjoM3NQt3wR8djudu+A57QbgqY3aSpIWT98Q\nuQ24OslzwFXdOkneluT//aVVkt8F/gj4kSQvJvnZbtcnkjyZ5BhwJfALPeuRJM3Qlo+zNlNVrwDv\nmbD9a8C1Y+sf2KD/B/ucX5I0X35jXZLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlS\nM0NEktTMEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlS\nM0NEktTMEJEkNTNEJEnNeoVIknOSPJjkue7nmye0+YEk/y3Jl5I8neRfbqe/JGlx9b0TOQg8XFUX\nAw936+v9GfD3quodwKXANUnetY3+kqQF1TdE9gN3d8t3A9evb1BrVrvVN3Svmra/JGlx7e3Zf6mq\nTnbLLwFLkxol2QM8DvwwcEdVfWE7/btjHAAOACwtLTEajZqLXl1d7dV/FqapcYj30PcYiz6OcOZc\n73mbVY2LMK932m643tPaMkSSPAS8dcKuW8dXqqqS1IR2VNV3gUuTvAm4N8nfqqqnpu3f7T8EHAJY\nXl6ulZWVrUrf0Gg0ok//Wdi0xgfuB+j3HvoeY4gaZmTXX+8FseM1LsK8npHdcL2ntWWIVNVVG+1L\n8nKSfVV1Msk+4NQWx/pfSR4BrgGeArbVX5K0WPp+JnIYuKlbvgn47PoGSd7S3YGQ5C8BVwPPTNtf\nkrS4+obIbcDVSZ4DrurWSfK2JEe6NvuAR5IcAx4DHqyq+zbrL0naHXp9sF5VrwDvmbD9a8C13fIx\n4Ee301+StDv4jXVJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSM0NEktTM\nEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSM0NEktSs\nV4gkOSfJg0me636+eZO2e5L89yT3jW37WJITSZ7oXtf2qUeSNFt970QOAg9X1cXAw936Rn4e+MqE\n7b9aVZd2ryM965EkzVDfENkP3N0t3w1cP6lRkvOB64Df6Hk+SdIC6RsiS1V1slt+CVjaoN2vAb8E\nfG/Cvo8kOZbkrs0eh0mSFs/erRokeQh464Rdt46vVFUlqQn9fxI4VVWPJ1lZt/tTwMeB6n5+Evjw\nBnUcAA4ALC0tMRqNtip9Q6urq736z8I0NQ7xHvoeY9HHEc6c6z1vs6pxEeb1TtsN13taW4ZIVV21\n0b4kLyfZV1Unk+wDTk1o9neA93cfmv8A8INJ/kNV/XRVvTx2rE8D903of7qOQ8AhgOXl5VpZWdmq\n9A2NRiP69J+FTWt84H6Afu+h7zGGqGFGdv31XhA7XuMizOsZ2Q3Xe1p9H2cdBm7qlm8CPru+QVX9\ni6o6v6ouBG4E/ktV/TRAFzyn3QA81bMeSdIM9Q2R24CrkzwHXNWtk+RtSab5S6tPJHkyyTHgSuAX\netYjSZqhLR9nbaaqXgHeM2H714DXfeejqkbAaGz9g33OL0maL7+xLklqZohIkpoZIpKkZoaIJKmZ\nISJJamaISJKa9foTX83HC7ddN/djvHDbdWfM/7ZBi2Goee28nC3vRCRJzQwRSVIzQ0SS1MwQkSQ1\nM0QkSc0MEUlSM0NEktTMEJEkNTNEJEnNUlXzrmHbkvxP4H/0OMS5wNcHKmenWONwdkOd1jic3VDn\nbqjxr1fVW7ZqtCtDpK8kR6tqed51bMYah7Mb6rTG4eyGOndDjdPycZYkqZkhIklq9v0aIofmXcAU\nrHE4u6FOaxzObqhzN9Q4le/Lz0QkScP4fr0TkSQN4IwNkSTXJHk2yfEkByfsT5Jf7/YfS3LZHGq8\nIMkjSb6c5OkkPz+hzUqSbyZ5onv9yhzqfCHJk935j07YP9exTPIjY+PzRJJvJfnoujZzGcckdyU5\nleSpsW3nJHkwyXPdzzdv0HfTObzDNf6rJM901/PeJG/aoO+mc2MGdX4syYmx63rtBn3nOZa/N1bf\nC0me2KDvzMZyUFV1xr2APcCfAG8HzgK+BFyyrs21wB8AAd4FfGEOde4DLuuW3wj88YQ6V4D75jye\nLwDnbrJ/7mO57tq/xNrfuM99HIF3A5cBT41t+wRwsFs+CNy+wfvYdA7vcI3vBfZ2y7dPqnGauTGD\nOj8G/OIUc2JuY7lu/yeBX5n3WA75OlPvRC4HjlfV81X1KnAPsH9dm/3Ab9WazwNvSrJvlkVW1cmq\n+mK3/L+BrwDnzbKGgcx9LMe8B/iTqurzZdTBVNWjwDfWbd4P3N0t3w1cP6HrNHN4x2qsqj+sqte6\n1c8D5+/Eubdjg7GcxlzH8rQkAf4h8Ls7ce55OVND5Dzgq2PrL/L6f5ynaTMzSS4EfhT4woTdP9E9\nVviDJH9zpoWtKeChJI8nOTBh/yKN5Y1s/Es673E8bamqTnbLLwFLE9os0ph+mLU7zUm2mhuz8JHu\nut61waPBRRnLvwu8XFXPbbB/EcZy287UENlVkvxl4PeBj1bVt9bt/iLw16rqbwP/BvjMrOsDrqiq\nS4H3AT+X5N1zqGFLSc4C3g/8xwm7F2EcX6fWnmMs7J9IJrkVeA347Q2azHtufIq1x1SXAidZe1y0\nqD7A5nch8x7LJmdqiJwALhhbP7/btt02Oy7JG1gLkN+uqv+8fn9VfauqVrvlI8Abkpw7yxqr6kT3\n8xRwL2uPB8YtxFiy9sv3xap6ef2ORRjHMS+fftzX/Tw1oc3cxzTJzwA/CfzjLuxeZ4q5saOq6uWq\n+m5VfQ/49AbnX4Sx3Av8feD3Nmoz77FsdaaGyGPAxUku6v7r9Ebg8Lo2h4EPdX9Z9C7gm2OPGGai\ne0b674CvVNW/3qDNW7t2JLmctWv2ygxrPDvJG08vs/aB61Prms19LDsb/pfevMdxncPATd3yTcBn\nJ7SZZg7vmCTXAL8EvL+q/nSDNtPMjR217rO3GzY4/1zHsnMV8ExVvThp5yKMZbN5f7K/Uy/W/mLo\nj1n7q4xbu203Azd3ywHu6PY/CSzPocYrWHuUcQx4ontdu67OW4CnWfuLks8DPzHjGt/enftLXR2L\nOpZnsxYKf2Vs29zHkbVQOwl8h7Vn8T8L/FXgYeA54CHgnK7t24Ajm83hGdZ4nLXPEU7PyzvX17jR\n3Jhxnf++m3PHWAuGfYs2lt323zw9F8fazm0sh3z5jXVJUrMz9XGWJGkGDBFJUjNDRJLUzBCRJDUz\nRCRJzQwRSVIzQ0SS1MwQkSQ1+78WQmpAGDyU2AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc21848ecc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEJCAYAAACOr7BbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGDZJREFUeJzt3X+sZGd93/H3p2tcgg2B4PRCdxd2UVdBWzCJuTIOdeFa\nQLtrEEsS1NoCEyjWyhIGEwWRJVEBiT8S2hQlNIbtFtyEgnAKOO0KLzaQeIQiY2tt49isjWFjKN6N\nwYRf5pooZsu3f8wxDONz75y9d2bujP1+SaOdc57nOfM9z52Zz54zv1JVSJI07J9sdAGSpNlkQEiS\nWhkQkqRWBoQkqZUBIUlqZUBIkloZEJKkVgaEJKmVASFJanXKRhfQ5owzzqht27ataewDDzzAaaed\nNt6CJmAe6rTG8ZmHOq1xfGa9zptvvvnvq+oXR3asqpm7PPe5z621uu6669Y8dprmoU5rHJ95qNMa\nx2fW6wRuqg7PxZ5ikiS1MiAkSa0MCElSKwNCktTKgJAktTIgJEmtDAhJUisDQpLUyoDQzNq272q2\n7bt6o8uQHrUMCElSKwNCktSqU0Ak2ZXkriRHk+xrad+T5LYktya5Kcm5zfqtSa5LckeSI0kuG/cO\nSJImY+S3uSbZBFwOvAQ4BhxOcrCq7hjo9pfAwaqqJGcC/wt4JnAC+O2quiXJ44Gbk3xmaKwkaQZ1\nOYI4GzhaVXdX1YPAlcCewQ5Vtdx8QyDAaUA16++tqlua6z8A7gQ2j6t4adJ8oVyPZvnp8/oKHZJX\nAruq6uJm+SLgeVV16VC/XwN+H/hnwEur6vND7duAzwHPqqr7W25nL7AXYGFh4blXXnnlmnZoeXmZ\n008/fU1jp2ke6tzoGl97zQMA/Omulb9Xf9I1dqmhi42eyy6scXxmvc7zzjvv5qpaHNlx1PeBA68E\nPjCwfBHwJ6v0fwHw2aF1pwM3A7/e5TvI/T2I2bDRNT79dz5ZT/+dT67aZ9I1dqmhi42eyy6scXxm\nvU7G+HsQx4GtA8tbmnUrBc7ngGckOQMgyWOATwAfqaqrOtyeJGkGdAmIw8COJNuTnApcABwc7JDk\nXyRJc/0s4J8C327WfRC4s6reM97SJUmTNPJdTFV1IsmlwLXAJuCKqjqS5JKmfT/wG8BrkvwI+Afg\n31dVNW93vQi4PcmtzSZ/t6oOTWJnJEnjMzIgAJon9END6/YPXH838O6WcX8NZJ01ag499M6fr/3B\nSze4Eklr5SepJUmtDAhJUisDQpLUyoCQJLUyICRJrQwISVIrA0KS1KrT5yD06POTzzEsbWwdkjaO\nRxCSpFYGhCSplQEhSWplQEiSWhkQkqRWBoQkqZUBIUlqZUBIklp1Cogku5LcleRokn0t7a9KcluS\n25Ncn+Q5A22/leRIki8m+WiSx45zByRJkzEyIJJsAi4HdgM7gQuT7Bzq9lXghVX1bOBdwIFm7Gbg\nTcBiVT2L/k+WXjC+8iVJk9LlCOJs4GhV3V1VDwJXAnsGO1TV9VX13WbxBmDLQPMpwM8lOQV4HPB3\n6y9bkjRpXQJiM3DPwPKxZt1KXg98CqCqjgN/CHwduBf4flV9em2lSpKmaaxf1pfkPPoBcW6z/CT6\nRxvbge8BH0vy6qr6cMvYvcBegIWFBXq93ppqWF5eXvPYaZqXOtdb4zj2cbVtTGse13sb8/D3tsbx\nmZc6R+kSEMeBrQPLW5p1PyPJmcAHgN1V9e1m9YuBr1bVt5o+VwHPBx4WEFV1gOa1i8XFxVpaWuq+\nFwN6vR5rHTtNM1/nNf1vc11zjesd33EbE5/HcewHc/D3xhrHaV7qHKXLKabDwI4k25OcSv9F5oOD\nHZI8DbgKuKiqvjzQ9HXgnCSPSxLgRcCd4yldkjRJI48gqupEkkuBa+m/C+mKqjqS5JKmfT/wduDJ\nwPv6OcCJqlqsqhuTfBy4BTgBfIHmKEGSNNs6vQZRVYeAQ0Pr9g9cvxi4eIWx7wDesY4aJUkbwE9S\nS5JaGRCSpFYGhCSplQEhSWplQEiSWhkQkqRWBsQj0LZ9V7Nt39UbXYakOWdASJJaGRCSpFYGhCSp\nlQGhRzRfj5HWzoCQpCmap/+0GBCSpFYGhCSplQEhSWplQEiSWhkQkqRWnQIiya4kdyU5mmRfS/ur\nktyW5PYk1yd5zlD7piRfSPLJcRUuSZqskQGRZBNwObAb2AlcmGTnULevAi+sqmcD7+Lhvzt9GXDn\n+suVJE1LlyOIs4GjVXV3VT0IXAnsGexQVddX1XebxRuALQ+1JdkCvBT4wHhKliRNwykd+mwG7hlY\nPgY8b5X+rwc+NbD8R8BbgcevdiNJ9gJ7ARYWFuj1eh1Ke7jl5eU1j52madQ5ju2vdxuTrqHrPG70\nfszD/dIax6dLnfOwH10CorMk59EPiHOb5ZcB91XVzUmWVhtbVQdoTk0tLi7W0tKq3VfU6/VY69hp\nmmid1/Q/pbmu7a93G1OqYeQ8zsJ+MB/3S2scn1XrHNN9ahq6BMRxYOvA8pZm3c9Icib900i7q+rb\nzep/Bbw8yfnAY4EnJPlwVb16fWVLkiaty2sQh4EdSbYnORW4ADg42CHJ04CrgIuq6ssPra+qt1XV\nlqra1oz7K8NBkubDyCOIqjqR5FLgWmATcEVVHUlySdO+H3g78GTgfUkATlTV4uTKliRNWqfXIKrq\nEHBoaN3+gesXAxeP2EYP6J10hZIEP/kG1K8tbWwdjyZ+klqS1MqAkKSO5um3HMbBgJCkOTLNkDIg\nJEmtDAhJUisDQpLUyoCQJLUyICRJrQwISVIrA0KS1MqAkCS1MiAkSa0MCElSq7H+opykh/NbSDWv\nPIKQJLUyICRJrToFRJJdSe5KcjTJvpb2VyW5LcntSa5P8pyuYyVJs2lkQCTZBFwO7AZ2Ahcm2TnU\n7avAC6vq2cC7gAMnMVaSNIO6HEGcDRytqrur6kHgSmDPYIequr6qvtss3gBs6TpWkjSbugTEZuCe\ngeVjzbqVvB741BrHSpJmxFjf5prkPPoBce4axu4F9gIsLCzQ6/XWVMPy8vKax07TNOocx/bXu41J\n19B1Hmd9P2bBvDx2ZqHGUTV0mctZuE+O0iUgjgNbB5a3NOt+RpIzgQ8Au6vq2yczFqCqDtC8drG4\nuFhLS0sdSnu4Xq/HWsdO00TrvKb/vvt1bX+925hSDSPncU72YxbM/GNnFuaxYw2rzuUs3Cc76nKK\n6TCwI8n2JKcCFwAHBzskeRpwFXBRVX35ZMZKkmbTyCOIqjqR5FLgWmATcEVVHUlySdO+H3g78GTg\nfUkATlTV4kpjJ7QvkrSqn3yq/Q9eusGVzIdOr0FU1SHg0NC6/QPXLwYu7jpWUnc+qWmj+ElqSVIr\nA0KS1MqAkCS1MiAkSa0MCElSKwNCktTKgJAktTIgJEmtDAhJU7Ft39U/+dCf5oMBIUlqZUBIkloZ\nEJKkVgaEJKmVASFJamVASJJaGRCSpFYGhCSpVaeASLIryV1JjibZ19L+zCSfT/KPSd4y1PbEJB9P\n8qUkdyb51XEV/0jlB4okzYKRPzmaZBNwOfAS4BhwOMnBqrpjoNt3gDcBr2jZxB8D11TVK5OcCjxu\n/WVLkiatyxHE2cDRqrq7qh4ErgT2DHaoqvuq6jDwo8H1SX4eeAHwwabfg1X1vbFULkmaqJFHEMBm\n4J6B5WPA8zpufzvwLeB/JHkOcDNwWVU9MNwxyV5gL8DCwgK9Xq/jTfys5eXlNY+dpi51rnc/xjEP\ns15D17/3rO/HtGpYzbQeO7MwD5OuYV4e36N0CYj1bv8s4I1VdWOSPwb2Af9xuGNVHQAOACwuLtbS\n0tKabrDX67HWsdO0ap3X9F9/WPN+rHf8HNUw8u89J/sx8Ro6mPhjZxbmYUo1zPzju6Mup5iOA1sH\nlrc067o4Bhyrqhub5Y/TDwxJ0ozrEhCHgR1JtjcvMl8AHOyy8ar6BnBPkl9qVr0IuGOVIZKkGTHy\nFFNVnUhyKXAtsAm4oqqOJLmkad+f5CnATcATgB8neTOws6ruB94IfKQJl7uB101oXyRJY9TpNYiq\nOgQcGlq3f+D6N+ifemobeyuwuI4aJUkbwE9SS5JaGRCSpFYGhCSplQEhSWplQEiSWhkQkqRWBoQk\nqZUBIUlqZUBIkloZEJKkVgaEJKmVASFJamVASJJaGRCSpFYGhCSplQEhSWrVKSCS7EpyV5KjSfa1\ntD8zyeeT/GOStwys35rkuiR3JDmS5LJxFj9s276ree01D0zyJiTpUWPkL8ol2QRcDrwEOAYcTnKw\nqgZ/W/o7wJuAVwwNPwH8dlXdkuTxwM1JPjM0VpI0g7ocQZwNHK2qu6vqQeBKYM9gh6q6r6oOAz8a\nWn9vVd3SXP8BcCeweSyVS5ImqktAbAbuGVg+xhqe5JNsA34FuPFkx0qSpm/kKaZxSHI68AngzVV1\n/wp99gJ7ARYWFuj1emu+vfWMnZbl5eWRda53P8YxD7NeQ5d5HEcdj5S5XE3XuVyvWZiHSdcwL4/v\nUboExHFg68DylmZdJ0keQz8cPlJVV63Ur6oOAAcAFhcXa2lpqetN/NQ1VwOwprFT1uv1Vq5zvfsx\njnmYkxpWncdx1PFImcsORs7les3CPEyphpl/fHfU5RTTYWBHku1JTgUuAA522XiSAB8E7qyq96y9\nTEnStI08gqiqE0kuBa4FNgFXVNWRJJc07fuTPAW4CXgC8OMkbwZ2AmcCFwG3J7m12eTvVtWhCeyL\nJGmMOr0G0TyhHxpat3/g+jfon3oa9tdA1lOgJGlj+Elq6VFg276r2bbv6o0uQ3PGgJAktTIgJEmt\nDAhJUisDYsw81yvpkcKAkCS1MiAkSa0MCElSKwNCktTKgJA0km++eHQyICRJrQwISVIrA0KS1MqA\nkCS1MiAkSa0MCElSKwNCktSqU0Ak2ZXkriRHk+xraU+S9zbttyU5a6Dtt5IcSfLFJB9N8thx7oAk\naTJGBkSSTcDlwG76vzN9YZKdQ912Azuay17g/c3YzcCbgMWqehb937S+YGzVS5ImpssRxNnA0aq6\nu6oeBK4E9gz12QN8qPpuAJ6Y5KlN2ynAzyU5BXgc8Hdjql2SNEGndOizGbhnYPkY8LwOfTZX1U1J\n/hD4OvAPwKer6tNtN5JkL/2jDxYWFuj1ep12oM16xo7LqBqWl5dH9lnvfoxjHma9hi7zOI46nMvp\n1DCN8dOoYV4e36N0CYg1S/Ik+kcX24HvAR9L8uqq+vBw36o6ABwAWFxcrKWlpZO/wWv63xWzprHj\n0rGGXq+3cp/17sc45mFOalh1HsdRh3M51RomOn6KNcz847ujLqeYjgNbB5a3NOu69Hkx8NWq+lZV\n/Qi4Cnj+2suVJE1Ll4A4DOxIsj3JqfRfZD441Ocg8Jrm3UznAN+vqnvpn1o6J8njkgR4EXDnGOuX\nJE3IyFNMVXUiyaXAtfTfhXRFVR1JcknTvh84BJwPHAV+CLyuabsxyceBW4ATwBdoTiNJkmZbp9cg\nquoQ/RAYXLd/4HoBb1hh7DuAd6yjRknSBvCT1JKkVgaEJKmVASFJamVASJJaGRBD/HF2SeozICRJ\nrQwISVIrA0KS1MqAkCS1MiAkSa0MCElSKwNCktTKgJAktTIgJEmtDAhJUisDQpLUqlNAJNmV5K4k\nR5Psa2lPkvc27bclOWuofVOSLyT55LgKlyRN1siASLIJuBzYDewELkyyc6jbbmBHc9kLvH+o/TL8\nLWpJmitdjiDOBo5W1d1V9SBwJbBnqM8e4EPVdwPwxCRPBUiyBXgp8IEx1i1JmrAuAbEZuGdg+Viz\nrmufPwLeCvx4jTVKkjbAKZPceJKXAfdV1c1Jlkb03Uv/9BQLCwv0er013+56xo5rG6PGLy8vj+wz\n6RqmsY1J19BlHsdRh3M5nRqmMX4aNczL43uULgFxHNg6sLylWdelz28AL09yPvBY4AlJPlxVrx6+\nkao6ABwAWFxcrKWlpa778FPX9H/oZ01jx7WNjuN7vd7KfaZUw0S3MaUaVp3HcdThXE61homOn2IN\nM//47qjLKabDwI4k25OcClwAHBzqcxB4TfNupnOA71fVvVX1tqraUlXbmnF/1RYOkqTZM/IIoqpO\nJLkUuBbYBFxRVUeSXNK07wcOAecDR4EfAq+bXMmSpGno9BpEVR2iHwKD6/YPXC/gDSO20QN6J12h\nJGlD+ElqSVIrA0KS1MqAkCS1MiAkSa0MCElSKwNCktTKgJAktTIgJEmtDAhJUisDQpLUyoCQJLUy\nICRJrQwISVIrA0KS1MqAkCS1MiAkSa0MCElSq04BkWRXkruSHE2yr6U9Sd7btN+W5KyuYyVJs2lk\nQCTZBFwO7AZ2Ahcm2TnUbTewo7nsBd5/EmMlSTMo/Z+TXqVD8qvAO6vq3zbLbwOoqt8f6PPfgF5V\nfbRZvgtYAraNGttmcXGxbrrppjXsTU5+jCTNoxHP3atJcnNVLY7qd0qHbW0G7hlYPgY8r0OfzR3H\nPlTwXvpHHywsLNDr9TqU9rOWTnqEJM2ntTxHnqwuATEVVXUAOAD9I4ilpaW1bIRer8eaxk7ZPNRp\njeMzD3Va4/hMo87Jbr2vS0AcB7YOLG9p1nXp85gOYyVJM6jLu5gOAzuSbE9yKnABcHCoz0HgNc27\nmc4Bvl9V93YcK0maQSOPIKrqRJJLgWuBTcAVVXUkySVN+37gEHA+cBT4IfC61cZOZE8kSWPV6TWI\nqjpEPwQG1+0fuF7AG7qOlSTNPj9JLUlqZUBIkloZEJKkVgaEJKmVASFJajXyu5g2QpJvAf93jcPP\nAP5+jOVMyjzUaY3jMw91WuP4zHqdT6+qXxzVaSYDYj2S3NTlS6g22jzUaY3jMw91WuP4zEudo3iK\nSZLUyoCQJLV6JAbEgY0uoKN5qNMax2ce6rTG8ZmXOlf1iHsNQpI0Ho/EIwhJ0hjMbUAk2ZXkriRH\nk+xraU+S9zbttyU5a8r1bU1yXZI7khxJcllLn6Uk309ya3N5+zRrHKjja0lub2p42G+9zsBc/tLA\nHN2a5P4kbx7qsyFzmeSKJPcl+eLAul9I8pkkX2n+fdIKY1e9D0+4xv+c5EvN3/MvkjxxhbGr3jcm\nXOM7kxwf+Juev8LYqczjKnX++UCNX0ty6wpjpzKXY1VVc3eh/9Xhfws8AzgV+Btg51Cf84FPAQHO\nAW6cco1PBc5qrj8e+HJLjUvAJ2dgPr8GnLFK+4bOZcvf/hv038e94XMJvAA4C/jiwLr/BOxrru8D\n3r3Cfqx6H55wjf8GOKW5/u62GrvcNyZc4zuBt3S4P0xlHleqc6j9vwBv38i5HOdlXo8gzgaOVtXd\nVfUgcCWwZ6jPHuBD1XcD8MQkT51WgVV1b1Xd0lz/AXAn/d/onkcbOpdDXgT8bVWt9YOUY1VVnwO+\nM7R6D/BnzfU/A17RMrTLfXhiNVbVp6vqRLN4A/1fe9wwK8xjF1ObR1i9ziQB/h3w0Und/rTNa0Bs\nBu4ZWD7Gw598u/SZiiTbgF8Bbmxpfn5zmP+pJP9yqoX9VAGfTXJzkr0t7TMzl/R/lXClB+AszCXA\nQvV/URH6RzsLLX1maU7/A/0jxDaj7huT9sbmb3rFCqfqZmke/zXwzar6ygrtGz2XJ21eA2JuJDkd\n+ATw5qq6f6j5FuBpVXUm8F+B/z3t+hrnVtUvA7uBNyR5wQbVsarmZ2tfDnyspXlW5vJnVP/cwsy+\nVTDJ7wEngI+s0GUj7xvvp3/q6JeBe+mfvpllF7L60cNcPM4GzWtAHAe2DixvadadbJ+JSvIY+uHw\nkaq6ari9qu6vquXm+iHgMUnOmGaNzW0fb/69D/gL+oftgzZ8Lhu7gVuq6pvDDbMyl41vPnQKrvn3\nvpY+Gz6nSV4LvAx4VRNkD9PhvjExVfXNqvp/VfVj4L+vcNsbPo8ASU4Bfh3485X6bORcrtW8BsRh\nYEeS7c3/Ki8ADg71OQi8pnkHzjnA9wcO+yeuOR/5QeDOqnrPCn2e0vQjydn0/x7fnlaNze2eluTx\nD12n/+LlF4e6behcDljxf2izMJcDDgK/2Vz/TeD/tPTpch+emCS7gLcCL6+qH67Qp8t9Y5I1Dr7O\n9Wsr3PaGzuOAFwNfqqpjbY0bPZdrttGvkq/1Qv+dNV+m/w6G32vWXQJc0lwPcHnTfjuwOOX6zqV/\nauE24Nbmcv5QjZcCR+i/8+IG4PkbMI/PaG7/b5paZm4umxpOo/+E//MD6zZ8LukH1r3Aj+if/349\n8GTgL4GvAJ8FfqHp+8+BQ6vdh6dY41H65+4fum/uH65xpfvGFGv8n8397Tb6T/pP3ch5XKnOZv2f\nPnRfHOi7IXM5zoufpJYktZrXU0ySpAkzICRJrQwISVIrA0KS1MqAkCS1MiAkSa0MCElSKwNCktTq\n/wPnApkY8Yp1sAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc21a6414a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._axes.Axes at 0x7fc218f2f860>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_i = 20\n",
    "\n",
    "data = np.array(inputs[batch_i][0])[:,[0]]\n",
    "labels = np.array(targets[batch_i][0])[:]\n",
    "predicted = np.array(outputs[batch_i][0])[:,[1]]\n",
    "#print(data)\n",
    "#print(labels)\n",
    "#print (predicted)\n",
    "gl.plot(np.array(range(data.size)), data, nf = 1)\n",
    "gl.stem(np.array(range(data.size)),labels - 0.5, nf = 1)\n",
    "gl.stem(np.array(range(data.size)),predicted - 0.5, nf = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
